{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/alxlnet/model_utils.py:334: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import xlnet\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tqdm import tqdm\n",
    "import model_utils\n",
    "import pickle\n",
    "import json\n",
    "pad_sequences = tf.keras.preprocessing.sequence.pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sentencepiece as spm\n",
    "from prepro_utils import preprocess_text, encode_ids\n",
    "\n",
    "sp_model = spm.SentencePieceProcessor()\n",
    "sp_model.Load('sp10m.cased.v9.model')\n",
    "\n",
    "def tokenize_fn(text):\n",
    "    text = preprocess_text(text, lower= False)\n",
    "    return encode_ids(sp_model, text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "SEG_ID_A   = 0\n",
    "SEG_ID_B   = 1\n",
    "SEG_ID_CLS = 2\n",
    "SEG_ID_SEP = 3\n",
    "SEG_ID_PAD = 4\n",
    "\n",
    "special_symbols = {\n",
    "    \"<unk>\"  : 0,\n",
    "    \"<s>\"    : 1,\n",
    "    \"</s>\"   : 2,\n",
    "    \"<cls>\"  : 3,\n",
    "    \"<sep>\"  : 4,\n",
    "    \"<pad>\"  : 5,\n",
    "    \"<mask>\" : 6,\n",
    "    \"<eod>\"  : 7,\n",
    "    \"<eop>\"  : 8,\n",
    "}\n",
    "\n",
    "VOCAB_SIZE = 32000\n",
    "UNK_ID = special_symbols[\"<unk>\"]\n",
    "CLS_ID = special_symbols[\"<cls>\"]\n",
    "SEP_ID = special_symbols[\"<sep>\"]\n",
    "MASK_ID = special_symbols[\"<mask>\"]\n",
    "EOD_ID = special_symbols[\"<eod>\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "with open('/home/husein/xlnet/train_X.pkl', 'rb') as fopen:\n",
    "    train_X, train_Y, train_depends, train_segments, train_masks = pickle.load(fopen)\n",
    "    \n",
    "with open('/home/husein/xlnet/test_X.pkl', 'rb') as fopen:\n",
    "    test_X, test_Y, test_depends, test_segments, test_masks = pickle.load(fopen)\n",
    "    \n",
    "with open('/home/husein/xlnet/tags.pkl', 'rb') as fopen:\n",
    "    idx2tag, tag2idx = pickle.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/alxlnet/xlnet.py:70: The name tf.gfile.Open is deprecated. Please use tf.io.gfile.GFile instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "kwargs = dict(\n",
    "      is_training=True,\n",
    "      use_tpu=False,\n",
    "      use_bfloat16=False,\n",
    "      dropout=0.1,\n",
    "      dropatt=0.1,\n",
    "      init='normal',\n",
    "      init_range=0.1,\n",
    "      init_std=0.05,\n",
    "      clamp_len=-1)\n",
    "\n",
    "xlnet_parameters = xlnet.RunConfig(**kwargs)\n",
    "xlnet_config = xlnet.XLNetConfig(\n",
    "    json_path = 'alxlnet-base-2020-04-10/config.json'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1173456 117345\n"
     ]
    }
   ],
   "source": [
    "epoch = 3\n",
    "batch_size = 8\n",
    "warmup_proportion = 0.1\n",
    "num_train_steps = int(len(train_X) / batch_size * epoch)\n",
    "num_warmup_steps = int(num_train_steps * warmup_proportion)\n",
    "print(num_train_steps, num_warmup_steps)\n",
    "\n",
    "training_parameters = dict(\n",
    "      decay_method = 'poly',\n",
    "      train_steps = num_train_steps,\n",
    "      learning_rate = 2e-5,\n",
    "      warmup_steps = num_warmup_steps,\n",
    "      min_lr_ratio = 0.0,\n",
    "      weight_decay = 0.00,\n",
    "      adam_epsilon = 1e-8,\n",
    "      num_core_per_host = 1,\n",
    "      lr_layer_decay_rate = 1,\n",
    "      use_tpu=False,\n",
    "      use_bfloat16=False,\n",
    "      dropout=0.0,\n",
    "      dropatt=0.0,\n",
    "      init='normal',\n",
    "      init_range=0.1,\n",
    "      init_std=0.02,\n",
    "      clip = 1.0,\n",
    "      clamp_len=-1,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Parameter:\n",
    "    def __init__(self, decay_method, warmup_steps, weight_decay, adam_epsilon, \n",
    "                num_core_per_host, lr_layer_decay_rate, use_tpu, learning_rate, train_steps,\n",
    "                min_lr_ratio, clip, **kwargs):\n",
    "        self.decay_method = decay_method\n",
    "        self.warmup_steps = warmup_steps\n",
    "        self.weight_decay = weight_decay\n",
    "        self.adam_epsilon = adam_epsilon\n",
    "        self.num_core_per_host = num_core_per_host\n",
    "        self.lr_layer_decay_rate = lr_layer_decay_rate\n",
    "        self.use_tpu = use_tpu\n",
    "        self.learning_rate = learning_rate\n",
    "        self.train_steps = train_steps\n",
    "        self.min_lr_ratio = min_lr_ratio\n",
    "        self.clip = clip\n",
    "        \n",
    "training_parameters = Parameter(**training_parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BiAAttention:\n",
    "    def __init__(self, input_size_encoder, input_size_decoder, num_labels):\n",
    "        self.input_size_encoder = input_size_encoder\n",
    "        self.input_size_decoder = input_size_decoder\n",
    "        self.num_labels = num_labels\n",
    "        \n",
    "        self.W_d = tf.get_variable(\"W_d\", shape=[self.num_labels, self.input_size_decoder],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.W_e = tf.get_variable(\"W_e\", shape=[self.num_labels, self.input_size_encoder],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.U = tf.get_variable(\"U\", shape=[self.num_labels, self.input_size_decoder, self.input_size_encoder],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        \n",
    "    def forward(self, input_d, input_e, mask_d=None, mask_e=None):\n",
    "        batch = tf.shape(input_d)[0]\n",
    "        length_decoder = tf.shape(input_d)[1]\n",
    "        length_encoder = tf.shape(input_e)[1]\n",
    "        out_d = tf.expand_dims(tf.matmul(self.W_d, tf.transpose(input_d, [0, 2, 1])), 3)\n",
    "        out_e = tf.expand_dims(tf.matmul(self.W_e, tf.transpose(input_e, [0, 2, 1])), 2)\n",
    "        output = tf.matmul(tf.expand_dims(input_d, 1), self.U)\n",
    "        output = tf.matmul(output, tf.transpose(tf.expand_dims(input_e, 1), [0, 1, 3, 2]))\n",
    "        \n",
    "        output = output + out_d + out_e\n",
    "        \n",
    "        if mask_d is not None:\n",
    "            d = tf.expand_dims(tf.expand_dims(mask_d, 1), 3)\n",
    "            e = tf.expand_dims(tf.expand_dims(mask_e, 1), 2)\n",
    "            output = output * d * e\n",
    "            \n",
    "        return output\n",
    "    \n",
    "class BiLinear:\n",
    "    def __init__(self, left_features, right_features, out_features):\n",
    "        self.left_features = left_features\n",
    "        self.right_features = right_features\n",
    "        self.out_features = out_features\n",
    "        \n",
    "        self.U = tf.get_variable(\"U-bi\", shape=[out_features, left_features, right_features],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.W_l = tf.get_variable(\"Wl\", shape=[out_features, left_features],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.W_r = tf.get_variable(\"Wr\", shape=[out_features, right_features],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "    \n",
    "    def forward(self, input_left, input_right):\n",
    "        left_size = tf.shape(input_left)\n",
    "        output_shape = tf.concat([left_size[:-1], [self.out_features]], axis = 0)\n",
    "        batch = tf.cast(tf.reduce_prod(left_size[:-1]), tf.int32)\n",
    "        input_left = tf.reshape(input_left, (batch, self.left_features))\n",
    "        input_right = tf.reshape(input_right, (batch, self.right_features))\n",
    "        tiled = tf.tile(tf.expand_dims(input_left, axis = 0), (self.out_features,1,1))\n",
    "        output = tf.transpose(tf.reduce_sum(tf.matmul(tiled, self.U), axis = 2))\n",
    "        output = output + tf.matmul(input_left, tf.transpose(self.W_l))\\\n",
    "        + tf.matmul(input_right, tf.transpose(self.W_r))\n",
    "        \n",
    "        return tf.reshape(output, output_shape)\n",
    "\n",
    "\n",
    "    \n",
    "    def decode(self, input_word, input_char, mask, leading_symbolic=0):\n",
    "        out_arc, out_type, _ = self.forward(input_word, input_char, mask)\n",
    "        batch = tf.shape(out_arc)[0]\n",
    "        max_len = tf.shape(out_arc)[1]\n",
    "        sec_max_len = tf.shape(out_arc)[2]\n",
    "        out_arc = out_arc + tf.linalg.diag(tf.fill([max_len], -np.inf))\n",
    "        minus_mask = tf.expand_dims(tf.cast(1 - mask, tf.bool), axis = 2)\n",
    "        minus_mask = tf.tile(minus_mask, [1, 1, sec_max_len])\n",
    "        out_arc = tf.where(minus_mask, tf.fill(tf.shape(out_arc), -np.inf), out_arc)\n",
    "        heads = tf.argmax(out_arc, axis = 1)\n",
    "        type_h, type_c = out_type\n",
    "        batch = tf.shape(type_h)[0]\n",
    "        max_len = tf.shape(type_h)[1]\n",
    "        batch_index = tf.range(0, batch)\n",
    "        t = tf.cast(tf.transpose(heads), tf.int32)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0), \n",
    "                                               tf.expand_dims(t, axis = 0)], axis = 0))\n",
    "        type_h = tf.gather_nd(type_h, concatenated)\n",
    "        out_type = self.bilinear.forward(type_h, type_c)\n",
    "        out_type = out_type[:, :, leading_symbolic:]\n",
    "        types = tf.argmax(out_type, axis = 2)\n",
    "        return heads, types\n",
    "    \n",
    "class Model:\n",
    "    def __init__(\n",
    "        self,\n",
    "        learning_rate,\n",
    "        hidden_size_word,\n",
    "        cov = 0.0):\n",
    "        \n",
    "        self.words = tf.placeholder(tf.int32, (None, None))\n",
    "        self.segment_ids = tf.placeholder(tf.int32, [None, None])\n",
    "        self.input_masks = tf.placeholder(tf.float32, [None, None])\n",
    "        self.heads = tf.placeholder(tf.int32, (None, None))\n",
    "        self.types = tf.placeholder(tf.int32, (None, None))\n",
    "        self.switch = tf.placeholder(tf.bool, None)\n",
    "        self.mask = tf.cast(tf.math.not_equal(self.words, 0), tf.float32)\n",
    "        self.maxlen = tf.shape(self.words)[1]\n",
    "        self.lengths = tf.count_nonzero(self.words, 1)\n",
    "        mask = self.mask\n",
    "        heads = self.heads\n",
    "        types = self.types\n",
    "        \n",
    "        self.arc_h = tf.layers.Dense(hidden_size_word)\n",
    "        self.arc_c = tf.layers.Dense(hidden_size_word)\n",
    "        self.attention = BiAAttention(hidden_size_word, hidden_size_word, 1)\n",
    "\n",
    "        self.type_h = tf.layers.Dense(hidden_size_word)\n",
    "        self.type_c = tf.layers.Dense(hidden_size_word)\n",
    "        self.bilinear = BiLinear(hidden_size_word, hidden_size_word, len(tag2idx))\n",
    "        \n",
    "        xlnet_model = xlnet.XLNetModel(\n",
    "            xlnet_config=xlnet_config,\n",
    "            run_config=xlnet_parameters,\n",
    "            input_ids=tf.transpose(self.words, [1, 0]),\n",
    "            seg_ids=tf.transpose(self.segment_ids, [1, 0]),\n",
    "            input_mask=tf.transpose(self.input_masks, [1, 0]))\n",
    "        output_layer = xlnet_model.get_sequence_output()\n",
    "        output_layer = tf.transpose(output_layer, [1, 0, 2])\n",
    "        \n",
    "        arc_h = tf.nn.elu(self.arc_h(output_layer))\n",
    "        arc_c = tf.nn.elu(self.arc_c(output_layer))\n",
    "        \n",
    "        type_h = tf.nn.elu(self.type_h(output_layer))\n",
    "        type_c = tf.nn.elu(self.type_c(output_layer))\n",
    "        \n",
    "        out_arc = tf.squeeze(self.attention.forward(arc_h, arc_c, mask_d=self.mask, \n",
    "                                                    mask_e=self.mask), axis = 1)\n",
    "        \n",
    "        batch = tf.shape(out_arc)[0]\n",
    "        max_len = tf.shape(out_arc)[1]\n",
    "        sec_max_len = tf.shape(out_arc)[2]\n",
    "        batch_index = tf.range(0, batch)\n",
    "        \n",
    "        decode_arc = out_arc + tf.linalg.diag(tf.fill([max_len], -np.inf))\n",
    "        minus_mask = tf.expand_dims(tf.cast(1 - mask, tf.bool), axis = 2)\n",
    "        minus_mask = tf.tile(minus_mask, [1, 1, sec_max_len])\n",
    "        decode_arc = tf.where(minus_mask, tf.fill(tf.shape(decode_arc), -np.inf), decode_arc)\n",
    "        self.heads_seq = tf.argmax(decode_arc, axis = 1)\n",
    "        self.heads_seq = tf.identity(self.heads_seq, name = 'heads_seq')\n",
    "        \n",
    "        t = tf.cast(tf.transpose(self.heads_seq), tf.int32)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0), \n",
    "                                               tf.expand_dims(t, axis = 0)], axis = 0))\n",
    "        type_h = tf.gather_nd(type_h, concatenated)\n",
    "        out_type = self.bilinear.forward(type_h, type_c)\n",
    "        self.tags_seq = tf.argmax(out_type, axis = 2)\n",
    "        self.tags_seq = tf.identity(self.tags_seq, name = 'tags_seq')\n",
    "        \n",
    "        log_likelihood, transition_params = tf.contrib.crf.crf_log_likelihood(\n",
    "            out_type, self.types, self.lengths\n",
    "        )\n",
    "        crf_loss = tf.reduce_mean(-log_likelihood)\n",
    "        self.logits, _ = tf.contrib.crf.crf_decode(\n",
    "            out_type, transition_params, self.lengths\n",
    "        )\n",
    "        self.logits = tf.identity(self.logits, name = 'logits')\n",
    "        \n",
    "        batch = tf.shape(out_arc)[0]\n",
    "        max_len = tf.shape(out_arc)[1]\n",
    "        batch_index = tf.range(0, batch)\n",
    "        t = tf.transpose(heads)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0), \n",
    "                                               tf.expand_dims(t, axis = 0)], axis = 0))\n",
    "        type_h = tf.gather_nd(type_h, concatenated)\n",
    "        out_type = self.bilinear.forward(type_h, type_c)\n",
    "        minus_inf = -1e8\n",
    "        minus_mask = (1 - mask) * minus_inf\n",
    "        out_arc = out_arc + tf.expand_dims(minus_mask, axis = 2) + tf.expand_dims(minus_mask, axis = 1)\n",
    "        loss_arc = tf.nn.log_softmax(out_arc, dim=1)\n",
    "        loss_type = tf.nn.log_softmax(out_type, dim=2)\n",
    "        loss_arc = loss_arc * tf.expand_dims(mask, axis = 2) * tf.expand_dims(mask, axis = 1)\n",
    "        loss_type = loss_type * tf.expand_dims(mask, axis = 2)\n",
    "        num = tf.reduce_sum(mask) - tf.cast(batch, tf.float32)\n",
    "        child_index = tf.tile(tf.expand_dims(tf.range(0, max_len), 1), [1, batch])\n",
    "        t = tf.transpose(heads)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0),\n",
    "                                               tf.expand_dims(t, axis = 0),\n",
    "                                               tf.expand_dims(child_index, axis = 0)], axis = 0))\n",
    "        loss_arc = tf.gather_nd(loss_arc, concatenated)\n",
    "        loss_arc = tf.transpose(loss_arc, [1, 0])\n",
    "        \n",
    "        t = tf.transpose(types)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0),\n",
    "                                               tf.expand_dims(child_index, axis = 0),\n",
    "                                               tf.expand_dims(t, axis = 0)], axis = 0))\n",
    "        loss_type = tf.gather_nd(loss_type, concatenated)\n",
    "        loss_type = tf.transpose(loss_type, [1, 0])\n",
    "        cost = (tf.reduce_sum(-loss_arc) / num) + (tf.reduce_sum(-loss_type) / num)\n",
    "        \n",
    "        self.cost = tf.cond(self.switch, lambda: cost + crf_loss, lambda: cost)\n",
    "        self.optimizer = tf.train.AdamOptimizer(\n",
    "            learning_rate = learning_rate\n",
    "        ).minimize(self.cost)\n",
    "        \n",
    "        mask = tf.sequence_mask(self.lengths, maxlen = self.maxlen)\n",
    "        \n",
    "        self.prediction = tf.boolean_mask(self.logits, mask)\n",
    "        mask_label = tf.boolean_mask(self.types, mask)\n",
    "        correct_pred = tf.equal(tf.cast(self.prediction, tf.int32), mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "        \n",
    "        self.prediction = tf.cast(tf.boolean_mask(self.heads_seq, mask), tf.int32)\n",
    "        mask_label = tf.boolean_mask(self.heads, mask)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy_depends = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "reduction_indices is deprecated, use axis instead\n",
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/alxlnet/xlnet.py:253: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/alxlnet/xlnet.py:253: The name tf.AUTO_REUSE is deprecated. Please use tf.compat.v1.AUTO_REUSE instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/alxlnet/custom_modeling.py:697: The name tf.logging.info is deprecated. Please use tf.compat.v1.logging.info instead.\n",
      "\n",
      "INFO:tensorflow:memory input None\n",
      "INFO:tensorflow:Use float type <dtype: 'float32'>\n",
      "WARNING:tensorflow:From /home/husein/alxlnet/custom_modeling.py:809: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dropout instead.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/layers/core.py:271: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.__call__` method instead.\n",
      "WARNING:tensorflow:From /home/husein/alxlnet/custom_modeling.py:109: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.Dense instead.\n",
      "WARNING:tensorflow:From <ipython-input-9-117b6be1eb3b>:138: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/contrib/crf/python/ops/crf.py:213: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
      "WARNING:tensorflow:From <ipython-input-9-117b6be1eb3b>:172: calling log_softmax (from tensorflow.python.ops.nn_ops) with dim is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "dim is deprecated, use axis instead\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "learning_rate = 2e-5\n",
    "hidden_size_word = 256\n",
    "\n",
    "model = Model(learning_rate, hidden_size_word)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "import re\n",
    "\n",
    "def get_assignment_map_from_checkpoint(tvars, init_checkpoint):\n",
    "    \"\"\"Compute the union of the current variables and checkpoint variables.\"\"\"\n",
    "    assignment_map = {}\n",
    "    initialized_variable_names = {}\n",
    "\n",
    "    name_to_variable = collections.OrderedDict()\n",
    "    for var in tvars:\n",
    "        name = var.name\n",
    "        m = re.match('^(.*):\\\\d+$', name)\n",
    "        if m is not None:\n",
    "            name = m.group(1)\n",
    "        name_to_variable[name] = var\n",
    "\n",
    "    init_vars = tf.train.list_variables(init_checkpoint)\n",
    "\n",
    "    assignment_map = collections.OrderedDict()\n",
    "    for x in init_vars:\n",
    "        (name, var) = (x[0], x[1])\n",
    "        if name not in name_to_variable:\n",
    "            continue\n",
    "        assignment_map[name] = name_to_variable[name]\n",
    "        initialized_variable_names[name] = 1\n",
    "        initialized_variable_names[name + ':0'] = 1\n",
    "\n",
    "    return (assignment_map, initialized_variable_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "tvars = tf.trainable_variables()\n",
    "checkpoint = 'alxlnet-base-2020-04-10/model.ckpt-300000'\n",
    "assignment_map, initialized_variable_names = get_assignment_map_from_checkpoint(tvars, \n",
    "                                                                                checkpoint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from alxlnet-base-2020-04-10/model.ckpt-300000\n"
     ]
    }
   ],
   "source": [
    "saver = tf.train.Saver(var_list = assignment_map)\n",
    "saver.restore(sess, checkpoint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "batch_x = train_X[:5]\n",
    "batch_x = pad_sequences(batch_x,padding='post')\n",
    "batch_y = train_Y[:5]\n",
    "batch_y = pad_sequences(batch_y,padding='post')\n",
    "batch_depends = train_depends[:5]\n",
    "batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "batch_segments = train_segments[:5]\n",
    "batch_segments = pad_sequences(batch_segments, padding='post', value = 4)\n",
    "batch_masks = train_masks[:5]\n",
    "batch_masks = pad_sequences(batch_masks, padding='post', value = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.08235294, 0.105882354, 40.901375]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess.run([model.accuracy, model.accuracy_depends, model.cost],\n",
    "        feed_dict = {model.words: batch_x,\n",
    "                model.types: batch_y,\n",
    "                model.heads: batch_depends,\n",
    "                model.segment_ids: batch_segments,\n",
    "                model.input_masks: batch_masks,\n",
    "                model.switch: False})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.03529412, 0.07058824, 185.67154]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess.run([model.accuracy, model.accuracy_depends, model.cost],\n",
    "        feed_dict = {model.words: batch_x,\n",
    "                model.types: batch_y,\n",
    "                model.heads: batch_depends,\n",
    "                model.segment_ids: batch_segments,\n",
    "                model.input_masks: batch_masks,\n",
    "                model.switch: True})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 391153/391153 [19:08:06<00:00,  5.68it/s, accuracy=0.933, accuracy_depends=0.667, cost=0.248]    \n",
      "test minibatch loop: 100%|██████████| 1250/1250 [02:10<00:00,  9.56it/s, accuracy=0.871, accuracy_depends=0.834, cost=0.396] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, training loss: 0.607109, training acc: 0.893016, training depends: 0.811151, valid loss: 0.446882, valid acc: 0.895227, valid depends: 0.837708\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop:  15%|█▌        | 59495/391153 [2:52:23<16:05:17,  5.73it/s, accuracy=0.918, accuracy_depends=0.826, cost=0.41]   IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop:  16%|█▌        | 62533/391153 [3:01:16<17:05:51,  5.34it/s, accuracy=0.89, accuracy_depends=0.884, cost=0.321]  IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop:  16%|█▋        | 64313/391153 [3:06:25<15:51:45,  5.72it/s, accuracy=0.918, accuracy_depends=0.874, cost=0.302]  IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop:  17%|█▋        | 67375/391153 [3:15:15<15:32:33,  5.79it/s, accuracy=0.902, accuracy_depends=0.853, cost=0.331] IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop:  18%|█▊        | 70411/391153 [3:24:09<16:20:56,  5.45it/s, accuracy=0.912, accuracy_depends=0.856, cost=0.559] IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop:  19%|█▊        | 73247/391153 [3:32:21<16:13:38,  5.44it/s, accuracy=0.908, accuracy_depends=0.853, cost=0.419] IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop:  20%|█▉        | 76413/391153 [3:42:30<14:28:51,  6.04it/s, accuracy=0.871, accuracy_depends=0.857, cost=0.57]  IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop:  20%|██        | 79403/391153 [3:51:15<16:15:16,  5.33it/s, accuracy=0.894, accuracy_depends=0.867, cost=0.412] IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop:  21%|██        | 82460/391153 [4:00:07<15:33:00,  5.51it/s, accuracy=0.919, accuracy_depends=0.823, cost=0.5]   IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop: 100%|██████████| 391153/391153 [19:10:06<00:00,  5.67it/s, accuracy=0.933, accuracy_depends=0.733, cost=0.234]    \n",
      "test minibatch loop: 100%|██████████| 1250/1250 [02:10<00:00,  9.58it/s, accuracy=0.902, accuracy_depends=0.834, cost=0.235] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, training loss: 0.429685, training acc: 0.896941, training depends: 0.841408, valid loss: 0.403095, valid acc: 0.901645, valid depends: 0.845275\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "epoch = 2\n",
    "for e in range(epoch):\n",
    "    train_acc, train_loss = [], []\n",
    "    test_acc, test_loss = [], []\n",
    "    train_acc_depends, test_acc_depends = [], []\n",
    "    \n",
    "    pbar = tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'train minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_X))\n",
    "        batch_x = train_X[i: index]\n",
    "        batch_x = pad_sequences(batch_x,padding='post')\n",
    "        batch_y = train_Y[i: index]\n",
    "        batch_y = pad_sequences(batch_y,padding='post')\n",
    "        batch_depends = train_depends[i: index]\n",
    "        batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "        batch_segments = train_segments[i: index]\n",
    "        batch_segments = pad_sequences(batch_segments, padding='post', value = 4)\n",
    "        batch_masks = train_masks[i: index]\n",
    "        batch_masks = pad_sequences(batch_masks, padding='post', value = 1)\n",
    "        \n",
    "        if batch_x.shape == batch_y.shape:\n",
    "        \n",
    "            acc_depends, acc, cost, _ = sess.run(\n",
    "                [model.accuracy_depends, model.accuracy, model.cost, model.optimizer],\n",
    "                feed_dict = {\n",
    "                    model.words: batch_x,\n",
    "                    model.types: batch_y,\n",
    "                    model.heads: batch_depends,\n",
    "                    model.segment_ids: batch_segments,\n",
    "                    model.input_masks: batch_masks,\n",
    "                    model.switch: False\n",
    "                },\n",
    "            )\n",
    "            train_loss.append(cost)\n",
    "            train_acc.append(acc)\n",
    "            train_acc_depends.append(acc_depends)\n",
    "            pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "        \n",
    "    pbar = tqdm(\n",
    "        range(0, len(test_X), batch_size), desc = 'test minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(test_X))\n",
    "        batch_x = test_X[i: index]\n",
    "        batch_x = pad_sequences(batch_x,padding='post')\n",
    "        batch_y = test_Y[i: index]\n",
    "        batch_y = pad_sequences(batch_y,padding='post')\n",
    "        batch_depends = test_depends[i: index]\n",
    "        batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "        batch_segments = test_segments[i: index]\n",
    "        batch_segments = pad_sequences(batch_segments, padding='post', value = 4)\n",
    "        batch_masks = test_masks[i: index]\n",
    "        batch_masks = pad_sequences(batch_masks, padding='post', value = 1)\n",
    "        \n",
    "        if batch_x.shape == batch_y.shape:\n",
    "        \n",
    "            acc_depends, acc, cost = sess.run(\n",
    "                [model.accuracy_depends, model.accuracy, model.cost],\n",
    "                feed_dict = {\n",
    "                    model.words: batch_x,\n",
    "                    model.types: batch_y,\n",
    "                    model.heads: batch_depends,\n",
    "                    model.segment_ids: batch_segments,\n",
    "                    model.input_masks: batch_masks,\n",
    "                    model.switch: False\n",
    "                },\n",
    "            )\n",
    "            test_loss.append(cost)\n",
    "            test_acc.append(acc)\n",
    "            test_acc_depends.append(acc_depends)\n",
    "            pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "    \n",
    "    \n",
    "    print(\n",
    "    'epoch: %d, training loss: %f, training acc: %f, training depends: %f, valid loss: %f, valid acc: %f, valid depends: %f\\n'\n",
    "    % (e, np.mean(train_loss), \n",
    "       np.mean(train_acc), \n",
    "       np.mean(train_acc_depends), \n",
    "       np.mean(test_loss), \n",
    "       np.mean(test_acc), \n",
    "       np.mean(test_acc_depends)\n",
    "    ))\n",
    "    \n",
    "    saver = tf.train.Saver(tf.trainable_variables())\n",
    "    saver.save(sess, 'alxlnet-base-dependency/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop:   1%|▏         | 5291/391153 [15:18<18:27:31,  5.81it/s, accuracy=0.952, accuracy_depends=0.85, cost=5.07]  IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop: 100%|██████████| 391153/391153 [18:50:23<00:00,  5.77it/s, accuracy=1, accuracy_depends=0.733, cost=0.427]       \n",
      "test minibatch loop: 100%|██████████| 1250/1250 [02:11<00:00,  9.52it/s, accuracy=0.969, accuracy_depends=0.853, cost=2.11] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, training loss: 3.915045, training acc: 0.947627, training depends: 0.841807, valid loss: 3.780288, valid acc: 0.949707, valid depends: 0.841579\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "epoch = 1\n",
    "for e in range(epoch):\n",
    "    train_acc, train_loss = [], []\n",
    "    test_acc, test_loss = [], []\n",
    "    train_acc_depends, test_acc_depends = [], []\n",
    "    \n",
    "    pbar = tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'train minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_X))\n",
    "        batch_x = train_X[i: index]\n",
    "        batch_x = pad_sequences(batch_x,padding='post')\n",
    "        batch_y = train_Y[i: index]\n",
    "        batch_y = pad_sequences(batch_y,padding='post')\n",
    "        batch_depends = train_depends[i: index]\n",
    "        batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "        batch_segments = train_segments[i: index]\n",
    "        batch_segments = pad_sequences(batch_segments, padding='post', value = 4)\n",
    "        batch_masks = train_masks[i: index]\n",
    "        batch_masks = pad_sequences(batch_masks, padding='post', value = 1)\n",
    "        \n",
    "        if batch_x.shape == batch_y.shape:\n",
    "        \n",
    "            acc_depends, acc, cost, _ = sess.run(\n",
    "                [model.accuracy_depends, model.accuracy, model.cost, model.optimizer],\n",
    "                feed_dict = {\n",
    "                    model.words: batch_x,\n",
    "                    model.types: batch_y,\n",
    "                    model.heads: batch_depends,\n",
    "                    model.segment_ids: batch_segments,\n",
    "                    model.input_masks: batch_masks,\n",
    "                    model.switch: True\n",
    "                },\n",
    "            )\n",
    "            train_loss.append(cost)\n",
    "            train_acc.append(acc)\n",
    "            train_acc_depends.append(acc_depends)\n",
    "            pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "        \n",
    "    pbar = tqdm(\n",
    "        range(0, len(test_X), batch_size), desc = 'test minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(test_X))\n",
    "        batch_x = test_X[i: index]\n",
    "        batch_x = pad_sequences(batch_x,padding='post')\n",
    "        batch_y = test_Y[i: index]\n",
    "        batch_y = pad_sequences(batch_y,padding='post')\n",
    "        batch_depends = test_depends[i: index]\n",
    "        batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "        batch_segments = test_segments[i: index]\n",
    "        batch_segments = pad_sequences(batch_segments, padding='post', value = 4)\n",
    "        batch_masks = test_masks[i: index]\n",
    "        batch_masks = pad_sequences(batch_masks, padding='post', value = 1)\n",
    "        \n",
    "        if batch_x.shape == batch_y.shape:\n",
    "        \n",
    "            acc_depends, acc, cost = sess.run(\n",
    "                [model.accuracy_depends, model.accuracy, model.cost],\n",
    "                feed_dict = {\n",
    "                    model.words: batch_x,\n",
    "                    model.types: batch_y,\n",
    "                    model.heads: batch_depends,\n",
    "                    model.segment_ids: batch_segments,\n",
    "                    model.input_masks: batch_masks,\n",
    "                    model.switch: True\n",
    "                },\n",
    "            )\n",
    "            test_loss.append(cost)\n",
    "            test_acc.append(acc)\n",
    "            test_acc_depends.append(acc_depends)\n",
    "            pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "    \n",
    "    \n",
    "    print(\n",
    "    'epoch: %d, training loss: %f, training acc: %f, training depends: %f, valid loss: %f, valid acc: %f, valid depends: %f\\n'\n",
    "    % (e, np.mean(train_loss), \n",
    "       np.mean(train_acc), \n",
    "       np.mean(train_acc_depends), \n",
    "       np.mean(test_loss), \n",
    "       np.mean(test_acc), \n",
    "       np.mean(test_acc_depends)\n",
    "    ))\n",
    "    \n",
    "    saver = tf.train.Saver(tf.trainable_variables())\n",
    "    saver.save(sess, 'alxlnet-base-dependency/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def merge_sentencepiece_tokens_tagging(x, y):\n",
    "    new_paired_tokens = []\n",
    "    n_tokens = len(x)\n",
    "    rejected = ['<cls>', '<sep>']\n",
    "\n",
    "    i = 0\n",
    "\n",
    "    while i < n_tokens:\n",
    "\n",
    "        current_token, current_label = x[i], y[i]\n",
    "        if not current_token.startswith('▁') and current_token not in rejected:\n",
    "            previous_token, previous_label = new_paired_tokens.pop()\n",
    "            merged_token = previous_token\n",
    "            merged_label = [previous_label]\n",
    "            while (\n",
    "                not current_token.startswith('▁')\n",
    "                and current_token not in rejected\n",
    "            ):\n",
    "                merged_token = merged_token + current_token.replace('▁', '')\n",
    "                merged_label.append(current_label)\n",
    "                i = i + 1\n",
    "                current_token, current_label = x[i], y[i]\n",
    "            merged_label = merged_label[0]\n",
    "            new_paired_tokens.append((merged_token, merged_label))\n",
    "\n",
    "        else:\n",
    "            new_paired_tokens.append((current_token, current_label))\n",
    "            i = i + 1\n",
    "\n",
    "    words = [\n",
    "        i[0].replace('▁', '')\n",
    "        for i in new_paired_tokens\n",
    "        if i[0] not in ['<cls>', '<sep>']\n",
    "    ]\n",
    "    labels = [i[1] for i in new_paired_tokens if i[0] not in ['<cls>', '<sep>']]\n",
    "    return words, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from unidecode import unidecode\n",
    "from malaya.function.parse_dependency import DependencyGraph\n",
    "\n",
    "PUNCTUATION = '!\"#$%&\\'()*+,./:;<=>?@[\\]^_`{|}~'\n",
    "\n",
    "def transformer_textcleaning(string):\n",
    "    \"\"\"\n",
    "    use by any transformer model before tokenization\n",
    "    \"\"\"\n",
    "    string = unidecode(string)\n",
    "    string = re.sub('\\\\(dot\\\\)', '.', string)\n",
    "    string = (\n",
    "        re.sub(re.findall(r'\\<a(.*?)\\>', string)[0], '', string)\n",
    "        if (len(re.findall(r'\\<a (.*?)\\>', string)) > 0)\n",
    "        and ('href' in re.findall(r'\\<a (.*?)\\>', string)[0])\n",
    "        else string\n",
    "    )\n",
    "    string = re.sub(\n",
    "        r'\\w+:\\/{2}[\\d\\w-]+(\\.[\\d\\w-]+)*(?:(?:\\/[^\\s/]*))*', ' ', string\n",
    "    )\n",
    "    string = re.sub(r'[ ]+', ' ', string).strip().split()\n",
    "    string = [w for w in string if w[0] != '@']\n",
    "    string = ' '.join(string)\n",
    "    string = re.sub(f'([{PUNCTUATION}])', r' \\1 ', string)\n",
    "    string = re.sub('\\s{2,}', ' ', string)\n",
    "    original_string = string.split()\n",
    "    string = [\n",
    "        (original_string[no], word.title() if word.isupper() else word)\n",
    "        for no, word in enumerate(string.split())\n",
    "        if len(word)\n",
    "    ]\n",
    "    return [s[0] for s in string], [s[1] for s in string]\n",
    "\n",
    "def parse_X(left):\n",
    "    left = ' '.join(left)\n",
    "    bert_tokens = tokenize_fn(left)\n",
    "    bert_tokens.extend([3, 4])\n",
    "    segment = [0] * (len(bert_tokens) - 1) + [SEG_ID_CLS]\n",
    "    input_mask = [0] * len(segment)\n",
    "    s_tokens = [sp_model.IdToPiece(i) for i in bert_tokens]\n",
    "    return bert_tokens, segment, input_mask, s_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dependency_graph(tagging, indexing):\n",
    "    \"\"\"\n",
    "    Return helper object for dependency parser results. Only accept tagging and indexing outputs from dependency models.\n",
    "    \"\"\"\n",
    "    result = []\n",
    "    for i in range(len(tagging)):\n",
    "        result.append(\n",
    "            '%d\\t%s\\t_\\t_\\t_\\t_\\t%d\\t%s\\t_\\t_'\n",
    "            % (i + 1, tagging[i][0], int(indexing[i][1]), tagging[i][1])\n",
    "        )\n",
    "    return DependencyGraph('\\n'.join(result), top_relation_label='root')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: G Pages: 1 -->\n",
       "<svg width=\"193pt\" height=\"218pt\"\n",
       " viewBox=\"0.00 0.00 192.50 218.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 214)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-214 188.5,-214 188.5,4 -4,4\"/>\n",
       "<!-- 0 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>0</title>\n",
       "<text text-anchor=\"middle\" x=\"118\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0 (None)</text>\n",
       "</g>\n",
       "<!-- 2 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>2</title>\n",
       "<text text-anchor=\"middle\" x=\"44\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">2 (makan)</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;2 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>0&#45;&gt;2</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M102.6672,-173.9735C92.0506,-161.4919 77.7592,-144.6899 65.94,-130.7944\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"68.4583,-128.3529 59.3132,-123.0034 63.1262,-132.8883 68.4583,-128.3529\"/>\n",
       "<text text-anchor=\"middle\" x=\"101\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">root</text>\n",
       "</g>\n",
       "<!-- 3 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>3</title>\n",
       "<text text-anchor=\"middle\" x=\"145\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">3 (ayam)</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;3 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>0&#45;&gt;3</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M123.5944,-173.9735C127.287,-162.0751 132.198,-146.2508 136.3846,-132.7606\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"139.7914,-133.5914 139.4127,-123.0034 133.106,-131.5166 139.7914,-133.5914\"/>\n",
       "<text text-anchor=\"middle\" x=\"144\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 1 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>1</title>\n",
       "<text text-anchor=\"middle\" x=\"44\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">1 (husein)</text>\n",
       "</g>\n",
       "<!-- 2&#45;&gt;1 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>2&#45;&gt;1</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M44,-86.9735C44,-75.1918 44,-59.5607 44,-46.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"47.5001,-46.0033 44,-36.0034 40.5001,-46.0034 47.5001,-46.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"64\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.files.Source at 0x7f51e5a1efd0>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string = 'husein makan ayam'\n",
    "sequence = transformer_textcleaning(string)[1]\n",
    "parsed_sequence, segment_sequence, mask_sequence, xlnet_sequence = parse_X(sequence)\n",
    "h, t = sess.run([model.heads_seq, model.tags_seq],\n",
    "        feed_dict = {\n",
    "            model.words: [parsed_sequence],\n",
    "            model.segment_ids: [segment_sequence],\n",
    "            model.input_masks: [mask_sequence],\n",
    "        },\n",
    ")\n",
    "h = h[0] - 2\n",
    "t = [idx2tag[d] for d in t[0]]\n",
    "merged_h = merge_sentencepiece_tokens_tagging(xlnet_sequence, h)\n",
    "merged_t = merge_sentencepiece_tokens_tagging(xlnet_sequence, t)\n",
    "tagging = list(zip(merged_t[0], merged_t[1]))\n",
    "indexing = list(zip(merged_h[0], merged_h[1]))\n",
    "dep = dependency_graph(tagging, indexing)\n",
    "dep.to_graphvis()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: G Pages: 1 -->\n",
       "<svg width=\"2322pt\" height=\"1175pt\"\n",
       " viewBox=\"0.00 0.00 2321.50 1175.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 1171)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-1171 2317.5,-1171 2317.5,4 -4,4\"/>\n",
       "<!-- 0 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>0</title>\n",
       "<text text-anchor=\"middle\" x=\"274.5\" y=\"-1145.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0 (None)</text>\n",
       "</g>\n",
       "<!-- 1 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>1</title>\n",
       "<text text-anchor=\"middle\" x=\"178.5\" y=\"-1058.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">1 (Kuala)</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;1 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>0&#45;&gt;1</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M254.6087,-1130.9735C240.5784,-1118.2586 221.601,-1101.0603 206.1064,-1087.0183\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"208.126,-1084.1251 198.3658,-1080.0034 203.4253,-1089.3121 208.126,-1084.1251\"/>\n",
       "<text text-anchor=\"middle\" x=\"247.5\" y=\"-1101.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">root</text>\n",
       "</g>\n",
       "<!-- 13 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>13</title>\n",
       "<text text-anchor=\"middle\" x=\"401.5\" y=\"-1058.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">13 (membidas)</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;13 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>0&#45;&gt;13</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M300.8145,-1130.9735C319.9714,-1117.8503 346.1,-1099.9512 366.939,-1085.6756\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"368.9474,-1088.5424 375.2192,-1080.0034 364.9913,-1082.7674 368.9474,-1088.5424\"/>\n",
       "<text text-anchor=\"middle\" x=\"380\" y=\"-1101.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">parataxis</text>\n",
       "</g>\n",
       "<!-- 2 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>2</title>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-971.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">2 (Lumpur)</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;2 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>1&#45;&gt;2</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M147.9078,-1043.9316C138.6872,-1038.326 128.6013,-1032.0332 119.5,-1026 106.776,-1017.5653 93.0898,-1007.8854 81.1619,-999.241\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"82.9682,-996.2264 72.8262,-993.1621 78.8436,-1001.8822 82.9682,-996.2264\"/>\n",
       "<text text-anchor=\"middle\" x=\"131.5\" y=\"-1014.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 3 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>3</title>\n",
       "<text text-anchor=\"middle\" x=\"142.5\" y=\"-971.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">3 (:)</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;3 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>1&#45;&gt;3</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M162.4111,-1043.9867C158.2797,-1038.5328 154.2347,-1032.3077 151.5,-1026 148.402,-1018.8542 146.3701,-1010.7049 145.0375,-1003.0494\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"148.4898,-1002.4657 143.6194,-993.0567 141.5593,-1003.4493 148.4898,-1002.4657\"/>\n",
       "<text text-anchor=\"middle\" x=\"172\" y=\"-1014.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 7 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>7</title>\n",
       "<text text-anchor=\"middle\" x=\"214.5\" y=\"-971.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">7 (,)</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;7 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>1&#45;&gt;7</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M185.9592,-1043.9735C190.931,-1031.9585 197.5593,-1015.9401 203.1767,-1002.3646\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"206.4608,-1003.5818 207.0503,-993.0034 199.9927,-1000.9053 206.4608,-1003.5818\"/>\n",
       "<text text-anchor=\"middle\" x=\"220\" y=\"-1014.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 4 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>4</title>\n",
       "<text text-anchor=\"middle\" x=\"300.5\" y=\"-971.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">4 (Ketua)</text>\n",
       "</g>\n",
       "<!-- 13&#45;&gt;4 -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>13&#45;&gt;4</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M380.5727,-1043.9735C365.6762,-1031.1419 345.4791,-1013.7443 329.0965,-999.6326\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"331.2614,-996.878 321.4005,-993.0034 326.6929,-1002.1817 331.2614,-996.878\"/>\n",
       "<text text-anchor=\"middle\" x=\"368.5\" y=\"-1014.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obl</text>\n",
       "</g>\n",
       "<!-- 8 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>8</title>\n",
       "<text text-anchor=\"middle\" x=\"401.5\" y=\"-971.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">8 (Datuk)</text>\n",
       "</g>\n",
       "<!-- 13&#45;&gt;8 -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>13&#45;&gt;8</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M401.5,-1043.9735C401.5,-1032.1918 401.5,-1016.5607 401.5,-1003.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"405.0001,-1003.0033 401.5,-993.0034 398.0001,-1003.0034 405.0001,-1003.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"421.5\" y=\"-1014.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 33 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>33</title>\n",
       "<text text-anchor=\"middle\" x=\"536.5\" y=\"-971.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">33 (melaksanakan)</text>\n",
       "</g>\n",
       "<!-- 13&#45;&gt;33 -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>13&#45;&gt;33</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M429.4721,-1043.9735C449.9263,-1030.792 477.8573,-1012.792 500.057,-998.4855\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"502.054,-1001.3624 508.5637,-993.0034 498.2621,-995.4784 502.054,-1001.3624\"/>\n",
       "<text text-anchor=\"middle\" x=\"496.5\" y=\"-1014.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">advcl</text>\n",
       "</g>\n",
       "<!-- 37 -->\n",
       "<g id=\"node16\" class=\"node\">\n",
       "<title>37</title>\n",
       "<text text-anchor=\"middle\" x=\"657.5\" y=\"-971.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">37 (.)</text>\n",
       "</g>\n",
       "<!-- 13&#45;&gt;37 -->\n",
       "<g id=\"edge15\" class=\"edge\">\n",
       "<title>13&#45;&gt;37</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M463.3032,-1045.3601C507.6821,-1032.7305 568.577,-1013.9974 620.5,-993 620.5993,-992.9598 620.6988,-992.9195 620.7983,-992.879\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"622.431,-995.9859 630.21,-988.793 619.6433,-989.5649 622.431,-995.9859\"/>\n",
       "<text text-anchor=\"middle\" x=\"588\" y=\"-1014.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 5 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>5</title>\n",
       "<text text-anchor=\"middle\" x=\"286.5\" y=\"-884.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">5 (Penerangan)</text>\n",
       "</g>\n",
       "<!-- 4&#45;&gt;5 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>4&#45;&gt;5</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M297.5992,-956.9735C295.7033,-945.1918 293.1879,-929.5607 291.0312,-916.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"294.4415,-915.3203 289.3971,-906.0034 287.5304,-916.4324 294.4415,-915.3203\"/>\n",
       "<text text-anchor=\"middle\" x=\"306.5\" y=\"-927.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 6 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>6</title>\n",
       "<text text-anchor=\"middle\" x=\"286.5\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">6 (Bersatu)</text>\n",
       "</g>\n",
       "<!-- 5&#45;&gt;6 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>5&#45;&gt;6</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M286.5,-869.9735C286.5,-858.1918 286.5,-842.5607 286.5,-829.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"290.0001,-829.0033 286.5,-819.0034 283.0001,-829.0034 290.0001,-829.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"298.5\" y=\"-840.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 9 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>9</title>\n",
       "<text text-anchor=\"middle\" x=\"402.5\" y=\"-884.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">9 (Wan)</text>\n",
       "</g>\n",
       "<!-- 8&#45;&gt;9 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>8&#45;&gt;9</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M401.7072,-956.9735C401.8426,-945.1918 402.0223,-929.5607 402.1763,-916.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"405.6778,-916.043 402.2931,-906.0034 398.6783,-915.9624 405.6778,-916.043\"/>\n",
       "<text text-anchor=\"middle\" x=\"414.5\" y=\"-927.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 10 -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>10</title>\n",
       "<text text-anchor=\"middle\" x=\"402.5\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">10 (Saiful)</text>\n",
       "</g>\n",
       "<!-- 9&#45;&gt;10 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>9&#45;&gt;10</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M402.5,-869.9735C402.5,-858.1918 402.5,-842.5607 402.5,-829.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"406.0001,-829.0033 402.5,-819.0034 399.0001,-829.0034 406.0001,-829.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"414.5\" y=\"-840.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 11 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>11</title>\n",
       "<text text-anchor=\"middle\" x=\"402.5\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">11 (Wan)</text>\n",
       "</g>\n",
       "<!-- 10&#45;&gt;11 -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>10&#45;&gt;11</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M402.5,-782.9735C402.5,-771.1918 402.5,-755.5607 402.5,-742.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"406.0001,-742.0033 402.5,-732.0034 399.0001,-742.0034 406.0001,-742.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"414.5\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 12 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>12</title>\n",
       "<text text-anchor=\"middle\" x=\"402.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">12 (Jan)</text>\n",
       "</g>\n",
       "<!-- 11&#45;&gt;12 -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>11&#45;&gt;12</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M402.5,-695.9735C402.5,-684.1918 402.5,-668.5607 402.5,-655.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"406.0001,-655.0033 402.5,-645.0034 399.0001,-655.0034 406.0001,-655.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"414.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 34 -->\n",
       "<g id=\"node36\" class=\"node\">\n",
       "<title>34</title>\n",
       "<text text-anchor=\"middle\" x=\"535.5\" y=\"-884.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">34 (sekatan)</text>\n",
       "</g>\n",
       "<!-- 33&#45;&gt;34 -->\n",
       "<g id=\"edge34\" class=\"edge\">\n",
       "<title>33&#45;&gt;34</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M536.2928,-956.9735C536.1574,-945.1918 535.9777,-929.5607 535.8237,-916.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"539.3217,-915.9624 535.7069,-906.0034 532.3222,-916.043 539.3217,-915.9624\"/>\n",
       "<text text-anchor=\"middle\" x=\"547.5\" y=\"-927.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 39 -->\n",
       "<g id=\"node37\" class=\"node\">\n",
       "<title>39</title>\n",
       "<text text-anchor=\"middle\" x=\"889.5\" y=\"-884.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">39 (berkata)</text>\n",
       "</g>\n",
       "<!-- 33&#45;&gt;39 -->\n",
       "<g id=\"edge35\" class=\"edge\">\n",
       "<title>33&#45;&gt;39</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M609.6418,-956.9735C674.0128,-941.1087 766.7037,-918.2643 827.5027,-903.2798\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"828.478,-906.6442 837.3499,-900.8529 826.8028,-899.8476 828.478,-906.6442\"/>\n",
       "<text text-anchor=\"middle\" x=\"748\" y=\"-927.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">dep</text>\n",
       "</g>\n",
       "<!-- 14 -->\n",
       "<g id=\"node17\" class=\"node\">\n",
       "<title>14</title>\n",
       "<text text-anchor=\"middle\" x=\"1339.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">14 (kenyataan)</text>\n",
       "</g>\n",
       "<!-- 16 -->\n",
       "<g id=\"node18\" class=\"node\">\n",
       "<title>16</title>\n",
       "<text text-anchor=\"middle\" x=\"1348.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">16 (Seri)</text>\n",
       "</g>\n",
       "<!-- 14&#45;&gt;16 -->\n",
       "<g id=\"edge16\" class=\"edge\">\n",
       "<title>14&#45;&gt;16</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1341.3648,-608.9735C1342.5836,-597.1918 1344.2006,-581.5607 1345.5871,-568.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1349.0899,-568.3105 1346.6376,-558.0034 1342.1271,-567.5901 1349.0899,-568.3105\"/>\n",
       "<text text-anchor=\"middle\" x=\"1356.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 15 -->\n",
       "<g id=\"node19\" class=\"node\">\n",
       "<title>15</title>\n",
       "<text text-anchor=\"middle\" x=\"937.5\" y=\"-1145.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">15 (Datuk)</text>\n",
       "</g>\n",
       "<!-- 15&#45;&gt;15 -->\n",
       "<g id=\"edge17\" class=\"edge\">\n",
       "<title>15&#45;&gt;15</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M983.5242,-1156.6827C993.9063,-1155.9906 1001.5,-1153.4297 1001.5,-1149 1001.5,-1146.2314 998.5337,-1144.1929 993.7743,-1142.8843\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"993.9383,-1139.3688 983.5242,-1141.3173 992.8804,-1146.2884 993.9383,-1139.3688\"/>\n",
       "<text text-anchor=\"middle\" x=\"1013.5\" y=\"-1145.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 17 -->\n",
       "<g id=\"node20\" class=\"node\">\n",
       "<title>17</title>\n",
       "<text text-anchor=\"middle\" x=\"829.5\" y=\"-1058.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">17 (Najib)</text>\n",
       "</g>\n",
       "<!-- 15&#45;&gt;17 -->\n",
       "<g id=\"edge18\" class=\"edge\">\n",
       "<title>15&#45;&gt;17</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M915.1223,-1130.9735C899.1934,-1118.1419 877.5964,-1100.7443 860.0784,-1086.6326\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"861.8322,-1083.551 851.849,-1080.0034 857.4409,-1089.0023 861.8322,-1083.551\"/>\n",
       "<text text-anchor=\"middle\" x=\"902.5\" y=\"-1101.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 20 -->\n",
       "<g id=\"node21\" class=\"node\">\n",
       "<title>20</title>\n",
       "<text text-anchor=\"middle\" x=\"937.5\" y=\"-1058.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">20 (Ketua)</text>\n",
       "</g>\n",
       "<!-- 15&#45;&gt;20 -->\n",
       "<g id=\"edge19\" class=\"edge\">\n",
       "<title>15&#45;&gt;20</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M937.5,-1130.9735C937.5,-1119.1918 937.5,-1103.5607 937.5,-1090.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"941.0001,-1090.0033 937.5,-1080.0034 934.0001,-1090.0034 941.0001,-1090.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"952.5\" y=\"-1101.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">conj</text>\n",
       "</g>\n",
       "<!-- 24 -->\n",
       "<g id=\"node22\" class=\"node\">\n",
       "<title>24</title>\n",
       "<text text-anchor=\"middle\" x=\"1155.5\" y=\"-1058.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">24 (Datuk)</text>\n",
       "</g>\n",
       "<!-- 15&#45;&gt;24 -->\n",
       "<g id=\"edge20\" class=\"edge\">\n",
       "<title>15&#45;&gt;24</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M982.6697,-1130.9735C1017.0148,-1117.267 1064.4143,-1098.3507 1100.8887,-1083.7944\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1102.3977,-1086.9607 1110.3881,-1080.0034 1099.8031,-1080.4593 1102.3977,-1086.9607\"/>\n",
       "<text text-anchor=\"middle\" x=\"1080.5\" y=\"-1101.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 19 -->\n",
       "<g id=\"node24\" class=\"node\">\n",
       "<title>19</title>\n",
       "<text text-anchor=\"middle\" x=\"786.5\" y=\"-971.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">19 (dan)</text>\n",
       "</g>\n",
       "<!-- 20&#45;&gt;19 -->\n",
       "<g id=\"edge21\" class=\"edge\">\n",
       "<title>20&#45;&gt;19</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M906.2127,-1043.9735C883.1319,-1030.6753 851.5394,-1012.473 826.6045,-998.1065\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"828.1593,-994.963 817.7472,-993.0034 824.6646,-1001.0283 828.1593,-994.963\"/>\n",
       "<text text-anchor=\"middle\" x=\"879.5\" y=\"-1014.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">cc</text>\n",
       "</g>\n",
       "<!-- 22 -->\n",
       "<g id=\"node25\" class=\"node\">\n",
       "<title>22</title>\n",
       "<text text-anchor=\"middle\" x=\"888.5\" y=\"-971.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">22 (Umno)</text>\n",
       "</g>\n",
       "<!-- 20&#45;&gt;22 -->\n",
       "<g id=\"edge22\" class=\"edge\">\n",
       "<title>20&#45;&gt;22</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M927.3472,-1043.9735C920.5144,-1031.8418 911.3828,-1015.6287 903.6899,-1001.9698\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"906.5968,-999.9989 898.6398,-993.0034 900.4977,-1003.4341 906.5968,-999.9989\"/>\n",
       "<text text-anchor=\"middle\" x=\"928.5\" y=\"-1014.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 25 -->\n",
       "<g id=\"node26\" class=\"node\">\n",
       "<title>25</title>\n",
       "<text text-anchor=\"middle\" x=\"986.5\" y=\"-971.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">25 (Dr)</text>\n",
       "</g>\n",
       "<!-- 20&#45;&gt;25 -->\n",
       "<g id=\"edge23\" class=\"edge\">\n",
       "<title>20&#45;&gt;25</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M947.6528,-1043.9735C954.4856,-1031.8418 963.6172,-1015.6287 971.3101,-1001.9698\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"974.5023,-1003.4341 976.3602,-993.0034 968.4032,-999.9989 974.5023,-1003.4341\"/>\n",
       "<text text-anchor=\"middle\" x=\"977.5\" y=\"-1014.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 21 -->\n",
       "<g id=\"node27\" class=\"node\">\n",
       "<title>21</title>\n",
       "<text text-anchor=\"middle\" x=\"1092.5\" y=\"-971.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">21 (Pemuda)</text>\n",
       "</g>\n",
       "<!-- 24&#45;&gt;21 -->\n",
       "<g id=\"edge25\" class=\"edge\">\n",
       "<title>24&#45;&gt;21</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1142.4464,-1043.9735C1133.4924,-1031.6085 1121.4681,-1015.0036 1111.4613,-1001.1847\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1114.2369,-999.05 1105.5369,-993.0034 1108.5673,-1003.1556 1114.2369,-999.05\"/>\n",
       "<text text-anchor=\"middle\" x=\"1140.5\" y=\"-1014.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 23 -->\n",
       "<g id=\"node28\" class=\"node\">\n",
       "<title>23</title>\n",
       "<text text-anchor=\"middle\" x=\"1191.5\" y=\"-971.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">23 (,)</text>\n",
       "</g>\n",
       "<!-- 24&#45;&gt;23 -->\n",
       "<g id=\"edge26\" class=\"edge\">\n",
       "<title>24&#45;&gt;23</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1162.9592,-1043.9735C1167.931,-1031.9585 1174.5593,-1015.9401 1180.1767,-1002.3646\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1183.4608,-1003.5818 1184.0503,-993.0034 1176.9927,-1000.9053 1183.4608,-1003.5818\"/>\n",
       "<text text-anchor=\"middle\" x=\"1197\" y=\"-1014.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 30 -->\n",
       "<g id=\"node29\" class=\"node\">\n",
       "<title>30</title>\n",
       "<text text-anchor=\"middle\" x=\"1319.5\" y=\"-971.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">30 (mempertikaikan)</text>\n",
       "</g>\n",
       "<!-- 24&#45;&gt;30 -->\n",
       "<g id=\"edge27\" class=\"edge\">\n",
       "<title>24&#45;&gt;30</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1189.4809,-1043.9735C1214.7687,-1030.5587 1249.4639,-1012.1533 1276.6538,-997.7294\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1278.3689,-1000.7816 1285.5626,-993.0034 1275.0884,-994.5978 1278.3689,-1000.7816\"/>\n",
       "<text text-anchor=\"middle\" x=\"1258\" y=\"-1014.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">acl</text>\n",
       "</g>\n",
       "<!-- 18 -->\n",
       "<g id=\"node23\" class=\"node\">\n",
       "<title>18</title>\n",
       "<text text-anchor=\"middle\" x=\"1092.5\" y=\"-884.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">18 (Razak)</text>\n",
       "</g>\n",
       "<!-- 21&#45;&gt;18 -->\n",
       "<g id=\"edge24\" class=\"edge\">\n",
       "<title>21&#45;&gt;18</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1092.5,-956.9735C1092.5,-945.1918 1092.5,-929.5607 1092.5,-916.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1096.0001,-916.0033 1092.5,-906.0034 1089.0001,-916.0034 1096.0001,-916.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1104.5\" y=\"-927.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 29 -->\n",
       "<g id=\"node33\" class=\"node\">\n",
       "<title>29</title>\n",
       "<text text-anchor=\"middle\" x=\"1274.5\" y=\"-884.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">29 (yang)</text>\n",
       "</g>\n",
       "<!-- 30&#45;&gt;29 -->\n",
       "<g id=\"edge31\" class=\"edge\">\n",
       "<title>30&#45;&gt;29</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1310.176,-956.9735C1303.9009,-944.8418 1295.5148,-928.6287 1288.4499,-914.9698\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1291.5151,-913.2776 1283.8121,-906.0034 1285.2976,-916.4935 1291.5151,-913.2776\"/>\n",
       "<text text-anchor=\"middle\" x=\"1320.5\" y=\"-927.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 31 -->\n",
       "<g id=\"node34\" class=\"node\">\n",
       "<title>31</title>\n",
       "<text text-anchor=\"middle\" x=\"1390.5\" y=\"-884.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">31 (tindakan)</text>\n",
       "</g>\n",
       "<!-- 30&#45;&gt;31 -->\n",
       "<g id=\"edge32\" class=\"edge\">\n",
       "<title>30&#45;&gt;31</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1334.2112,-956.9735C1344.3974,-944.4919 1358.1094,-927.6899 1369.4494,-913.7944\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1372.1965,-915.9638 1375.8076,-906.0034 1366.7733,-911.5379 1372.1965,-915.9638\"/>\n",
       "<text text-anchor=\"middle\" x=\"1369.5\" y=\"-927.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 26 -->\n",
       "<g id=\"node30\" class=\"node\">\n",
       "<title>26</title>\n",
       "<text text-anchor=\"middle\" x=\"1470.5\" y=\"-1145.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">26 (Asyraf)</text>\n",
       "</g>\n",
       "<!-- 26&#45;&gt;26 -->\n",
       "<g id=\"edge28\" class=\"edge\">\n",
       "<title>26&#45;&gt;26</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1518.1902,-1156.6424C1528.5134,-1155.8951 1536,-1153.3477 1536,-1149 1536,-1146.2827 1533.0756,-1144.2686 1528.3619,-1142.9578\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1528.6126,-1139.4542 1518.1902,-1141.3576 1527.5248,-1146.3692 1528.6126,-1139.4542\"/>\n",
       "<text text-anchor=\"middle\" x=\"1548\" y=\"-1145.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 27 -->\n",
       "<g id=\"node31\" class=\"node\">\n",
       "<title>27</title>\n",
       "<text text-anchor=\"middle\" x=\"1470.5\" y=\"-1058.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">27 (Wajdi)</text>\n",
       "</g>\n",
       "<!-- 26&#45;&gt;27 -->\n",
       "<g id=\"edge29\" class=\"edge\">\n",
       "<title>26&#45;&gt;27</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1470.5,-1130.9735C1470.5,-1119.1918 1470.5,-1103.5607 1470.5,-1090.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1474.0001,-1090.0033 1470.5,-1080.0034 1467.0001,-1090.0034 1474.0001,-1090.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1482.5\" y=\"-1101.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 28 -->\n",
       "<g id=\"node32\" class=\"node\">\n",
       "<title>28</title>\n",
       "<text text-anchor=\"middle\" x=\"1470.5\" y=\"-971.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">28 (Dusuki)</text>\n",
       "</g>\n",
       "<!-- 27&#45;&gt;28 -->\n",
       "<g id=\"edge30\" class=\"edge\">\n",
       "<title>27&#45;&gt;28</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1470.5,-1043.9735C1470.5,-1032.1918 1470.5,-1016.5607 1470.5,-1003.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1474.0001,-1003.0033 1470.5,-993.0034 1467.0001,-1003.0034 1474.0001,-1003.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1482.5\" y=\"-1014.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 32 -->\n",
       "<g id=\"node35\" class=\"node\">\n",
       "<title>32</title>\n",
       "<text text-anchor=\"middle\" x=\"1390.5\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">32 (kerajaan)</text>\n",
       "</g>\n",
       "<!-- 31&#45;&gt;32 -->\n",
       "<g id=\"edge33\" class=\"edge\">\n",
       "<title>31&#45;&gt;32</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1390.5,-869.9735C1390.5,-858.1918 1390.5,-842.5607 1390.5,-829.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1394.0001,-829.0033 1390.5,-819.0034 1387.0001,-829.0034 1394.0001,-829.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1427.5\" y=\"-840.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">compound</text>\n",
       "</g>\n",
       "<!-- 35 -->\n",
       "<g id=\"node38\" class=\"node\">\n",
       "<title>35</title>\n",
       "<text text-anchor=\"middle\" x=\"532.5\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">35 (pergerakan)</text>\n",
       "</g>\n",
       "<!-- 34&#45;&gt;35 -->\n",
       "<g id=\"edge36\" class=\"edge\">\n",
       "<title>34&#45;&gt;35</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M534.8784,-869.9735C534.4721,-858.1918 533.9331,-842.5607 533.471,-829.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"536.9634,-828.8768 533.1208,-819.0034 529.9676,-829.1181 536.9634,-828.8768\"/>\n",
       "<text text-anchor=\"middle\" x=\"571.5\" y=\"-840.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">compound</text>\n",
       "</g>\n",
       "<!-- 46 -->\n",
       "<g id=\"node41\" class=\"node\">\n",
       "<title>46</title>\n",
       "<text text-anchor=\"middle\" x=\"908.5\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">46 (memetik)</text>\n",
       "</g>\n",
       "<!-- 39&#45;&gt;46 -->\n",
       "<g id=\"edge38\" class=\"edge\">\n",
       "<title>39&#45;&gt;46</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M893.4368,-869.9735C896.0098,-858.1918 899.4235,-842.5607 902.3505,-829.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"905.8539,-829.5199 904.5682,-819.0034 899.0151,-828.0263 905.8539,-829.5199\"/>\n",
       "<text text-anchor=\"middle\" x=\"914\" y=\"-840.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">dep</text>\n",
       "</g>\n",
       "<!-- 36 -->\n",
       "<g id=\"node39\" class=\"node\">\n",
       "<title>36</title>\n",
       "<text text-anchor=\"middle\" x=\"521.5\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">36 (penuh)</text>\n",
       "</g>\n",
       "<!-- 35&#45;&gt;36 -->\n",
       "<g id=\"edge37\" class=\"edge\">\n",
       "<title>35&#45;&gt;36</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M530.2208,-782.9735C528.7311,-771.1918 526.7548,-755.5607 525.0602,-742.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"528.5031,-741.4853 523.7763,-732.0034 521.5584,-742.3634 528.5031,-741.4853\"/>\n",
       "<text text-anchor=\"middle\" x=\"548.5\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">amod</text>\n",
       "</g>\n",
       "<!-- 38 -->\n",
       "<g id=\"node40\" class=\"node\">\n",
       "<title>38</title>\n",
       "<text text-anchor=\"middle\" x=\"634.5\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">38 (Beliau)</text>\n",
       "</g>\n",
       "<!-- 46&#45;&gt;38 -->\n",
       "<g id=\"edge42\" class=\"edge\">\n",
       "<title>46&#45;&gt;38</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M852.3063,-792.1031C819.7417,-786.1375 778.2417,-777.1375 742.5,-765 720.1262,-757.4021 696.2717,-746.4632 676.6879,-736.663\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"678.2283,-733.5198 667.7276,-732.1103 675.0574,-739.7605 678.2283,-733.5198\"/>\n",
       "<text text-anchor=\"middle\" x=\"762.5\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 40 -->\n",
       "<g id=\"node42\" class=\"node\">\n",
       "<title>40</title>\n",
       "<text text-anchor=\"middle\" x=\"727.5\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">40 (,)</text>\n",
       "</g>\n",
       "<!-- 46&#45;&gt;40 -->\n",
       "<g id=\"edge44\" class=\"edge\">\n",
       "<title>46&#45;&gt;40</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M854.9613,-782.8844C840.931,-777.6079 825.9661,-771.4982 812.5,-765 795.9425,-757.0101 778.4022,-746.8075 763.5825,-737.6303\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"765.0699,-734.4315 754.7399,-732.0777 761.3474,-740.3597 765.0699,-734.4315\"/>\n",
       "<text text-anchor=\"middle\" x=\"833\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 41 -->\n",
       "<g id=\"node43\" class=\"node\">\n",
       "<title>41</title>\n",
       "<text text-anchor=\"middle\" x=\"816.5\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">41 (Najib)</text>\n",
       "</g>\n",
       "<!-- 46&#45;&gt;41 -->\n",
       "<g id=\"edge43\" class=\"edge\">\n",
       "<title>46&#45;&gt;41</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M889.4375,-782.9735C875.9918,-770.2586 857.8051,-753.0603 842.9561,-739.0183\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"845.2086,-736.3312 835.538,-732.0034 840.399,-741.4173 845.2086,-736.3312\"/>\n",
       "<text text-anchor=\"middle\" x=\"888.5\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 45 -->\n",
       "<g id=\"node47\" class=\"node\">\n",
       "<title>45</title>\n",
       "<text text-anchor=\"middle\" x=\"930.5\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">45 (sengaja)</text>\n",
       "</g>\n",
       "<!-- 46&#45;&gt;45 -->\n",
       "<g id=\"edge46\" class=\"edge\">\n",
       "<title>46&#45;&gt;45</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M913.0584,-782.9735C916.0672,-771.0751 920.0688,-755.2508 923.4801,-741.7606\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"926.889,-742.5563 925.9474,-732.0034 920.1026,-740.8401 926.889,-742.5563\"/>\n",
       "<text text-anchor=\"middle\" x=\"950\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">advmod</text>\n",
       "</g>\n",
       "<!-- 61 -->\n",
       "<g id=\"node48\" class=\"node\">\n",
       "<title>61</title>\n",
       "<text text-anchor=\"middle\" x=\"1027.5\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">61 (.)</text>\n",
       "</g>\n",
       "<!-- 46&#45;&gt;61 -->\n",
       "<g id=\"edge45\" class=\"edge\">\n",
       "<title>46&#45;&gt;61</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M952.8254,-782.9686C963.0311,-777.8773 973.517,-771.8332 982.5,-765 991.7586,-757.9572 1000.5554,-748.7823 1007.8747,-740.1591\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1010.7997,-742.1104 1014.4064,-732.1483 1005.3745,-737.6869 1010.7997,-742.1104\"/>\n",
       "<text text-anchor=\"middle\" x=\"1019\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 47 -->\n",
       "<g id=\"node49\" class=\"node\">\n",
       "<title>47</title>\n",
       "<text text-anchor=\"middle\" x=\"1134.5\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">47 (kenyataan)</text>\n",
       "</g>\n",
       "<!-- 46&#45;&gt;47 -->\n",
       "<g id=\"edge47\" class=\"edge\">\n",
       "<title>46&#45;&gt;47</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M964.6972,-789.3444C989.3424,-783.3891 1018.3287,-775.2209 1043.5,-765 1062.0471,-757.4689 1081.5425,-746.9337 1097.7097,-737.3925\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1099.5517,-740.369 1106.3219,-732.2195 1095.9473,-734.3683 1099.5517,-740.369\"/>\n",
       "<text text-anchor=\"middle\" x=\"1086.5\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 64 -->\n",
       "<g id=\"node50\" class=\"node\">\n",
       "<title>64</title>\n",
       "<text text-anchor=\"middle\" x=\"1507.5\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">64 (berkata)</text>\n",
       "</g>\n",
       "<!-- 46&#45;&gt;64 -->\n",
       "<g id=\"edge48\" class=\"edge\">\n",
       "<title>46&#45;&gt;64</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M964.5833,-792.8544C1076.6622,-776.5758 1327.5956,-740.1297 1444.8644,-723.0973\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1445.7164,-726.5104 1455.1095,-721.6093 1444.7102,-719.5831 1445.7164,-726.5104\"/>\n",
       "<text text-anchor=\"middle\" x=\"1258\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">dep</text>\n",
       "</g>\n",
       "<!-- 43 -->\n",
       "<g id=\"node44\" class=\"node\">\n",
       "<title>43</title>\n",
       "<text text-anchor=\"middle\" x=\"816.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">43 (Asyraf)</text>\n",
       "</g>\n",
       "<!-- 41&#45;&gt;43 -->\n",
       "<g id=\"edge39\" class=\"edge\">\n",
       "<title>41&#45;&gt;43</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M816.5,-695.9735C816.5,-684.1918 816.5,-668.5607 816.5,-655.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"820.0001,-655.0033 816.5,-645.0034 813.0001,-655.0034 820.0001,-655.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"831.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">conj</text>\n",
       "</g>\n",
       "<!-- 42 -->\n",
       "<g id=\"node45\" class=\"node\">\n",
       "<title>42</title>\n",
       "<text text-anchor=\"middle\" x=\"720.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">42 (dan)</text>\n",
       "</g>\n",
       "<!-- 43&#45;&gt;42 -->\n",
       "<g id=\"edge40\" class=\"edge\">\n",
       "<title>43&#45;&gt;42</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M796.6087,-608.9735C782.5784,-596.2586 763.601,-579.0603 748.1064,-565.0183\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"750.126,-562.1251 740.3658,-558.0034 745.4253,-567.3121 750.126,-562.1251\"/>\n",
       "<text text-anchor=\"middle\" x=\"782.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">cc</text>\n",
       "</g>\n",
       "<!-- 44 -->\n",
       "<g id=\"node46\" class=\"node\">\n",
       "<title>44</title>\n",
       "<text text-anchor=\"middle\" x=\"821.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">44 (Wajdi)</text>\n",
       "</g>\n",
       "<!-- 43&#45;&gt;44 -->\n",
       "<g id=\"edge41\" class=\"edge\">\n",
       "<title>43&#45;&gt;44</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M817.536,-608.9735C818.2131,-597.1918 819.1115,-581.5607 819.8817,-568.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"823.3857,-568.1877 820.4653,-558.0034 816.3972,-567.786 823.3857,-568.1877\"/>\n",
       "<text text-anchor=\"middle\" x=\"831.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 48 -->\n",
       "<g id=\"node51\" class=\"node\">\n",
       "<title>48</title>\n",
       "<text text-anchor=\"middle\" x=\"1129.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">48 (Perdana)</text>\n",
       "</g>\n",
       "<!-- 47&#45;&gt;48 -->\n",
       "<g id=\"edge49\" class=\"edge\">\n",
       "<title>47&#45;&gt;48</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1133.464,-695.9735C1132.7869,-684.1918 1131.8885,-668.5607 1131.1183,-655.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1134.6028,-654.786 1130.5347,-645.0034 1127.6143,-655.1877 1134.6028,-654.786\"/>\n",
       "<text text-anchor=\"middle\" x=\"1144.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 64&#45;&gt;14 -->\n",
       "<g id=\"edge63\" class=\"edge\">\n",
       "<title>64&#45;&gt;14</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1472.6903,-695.9735C1446.7857,-682.5587 1411.2443,-664.1533 1383.3912,-649.7294\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1384.7546,-646.494 1374.2651,-645.0034 1381.5356,-652.7099 1384.7546,-646.494\"/>\n",
       "<text text-anchor=\"middle\" x=\"1444.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 62 -->\n",
       "<g id=\"node64\" class=\"node\">\n",
       "<title>62</title>\n",
       "<text text-anchor=\"middle\" x=\"1459.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">62 (Wan)</text>\n",
       "</g>\n",
       "<!-- 64&#45;&gt;62 -->\n",
       "<g id=\"edge64\" class=\"edge\">\n",
       "<title>64&#45;&gt;62</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1491.6037,-695.6612C1487.3319,-690.2021 1482.9463,-684.0558 1479.5,-678 1475.3325,-670.677 1471.6542,-662.3145 1468.6256,-654.5091\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1471.8813,-653.2212 1465.149,-645.0404 1465.3102,-655.634 1471.8813,-653.2212\"/>\n",
       "<text text-anchor=\"middle\" x=\"1499.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 65 -->\n",
       "<g id=\"node66\" class=\"node\">\n",
       "<title>65</title>\n",
       "<text text-anchor=\"middle\" x=\"1544.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">65 (,)</text>\n",
       "</g>\n",
       "<!-- 64&#45;&gt;65 -->\n",
       "<g id=\"edge65\" class=\"edge\">\n",
       "<title>64&#45;&gt;65</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1515.1664,-695.9735C1520.2763,-683.9585 1527.0887,-667.9401 1532.8622,-654.3646\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1536.1505,-655.5755 1536.8434,-645.0034 1529.7089,-652.8359 1536.1505,-655.5755\"/>\n",
       "<text text-anchor=\"middle\" x=\"1548\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 83 -->\n",
       "<g id=\"node67\" class=\"node\">\n",
       "<title>83</title>\n",
       "<text text-anchor=\"middle\" x=\"1616.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">83 (.)</text>\n",
       "</g>\n",
       "<!-- 64&#45;&gt;83 -->\n",
       "<g id=\"edge66\" class=\"edge\">\n",
       "<title>64&#45;&gt;83</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1545.3878,-695.9598C1554.6715,-690.7617 1564.2907,-684.6708 1572.5,-678 1581.3032,-670.8466 1589.7523,-661.7921 1596.8518,-653.3041\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1599.6572,-655.4009 1603.2124,-645.4204 1594.2092,-651.0054 1599.6572,-655.4009\"/>\n",
       "<text text-anchor=\"middle\" x=\"1607\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 68 -->\n",
       "<g id=\"node68\" class=\"node\">\n",
       "<title>68</title>\n",
       "<text text-anchor=\"middle\" x=\"1725.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">68 (menjangka)</text>\n",
       "</g>\n",
       "<!-- 64&#45;&gt;68 -->\n",
       "<g id=\"edge67\" class=\"edge\">\n",
       "<title>64&#45;&gt;68</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1559.6276,-701.5362C1582.1151,-695.4802 1608.4843,-687.4562 1631.5,-678 1650.3374,-670.2605 1670.2892,-659.7905 1686.9423,-650.3507\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1688.9063,-653.2586 1695.8266,-645.2362 1685.4139,-647.192 1688.9063,-653.2586\"/>\n",
       "<text text-anchor=\"middle\" x=\"1681.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">advcl</text>\n",
       "</g>\n",
       "<!-- 49 -->\n",
       "<g id=\"node52\" class=\"node\">\n",
       "<title>49</title>\n",
       "<text text-anchor=\"middle\" x=\"947.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">49 (Menteri)</text>\n",
       "</g>\n",
       "<!-- 48&#45;&gt;49 -->\n",
       "<g id=\"edge50\" class=\"edge\">\n",
       "<title>48&#45;&gt;49</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1084.6337,-608.8225C1071.5913,-603.3167 1057.3903,-597.1054 1044.5,-591 1026.2197,-582.3417 1006.3938,-572.0792 989.4586,-563.0382\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"990.8414,-559.808 980.3757,-558.159 987.5288,-565.9746 990.8414,-559.808\"/>\n",
       "<text text-anchor=\"middle\" x=\"1056.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 50 -->\n",
       "<g id=\"node53\" class=\"node\">\n",
       "<title>50</title>\n",
       "<text text-anchor=\"middle\" x=\"1046.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">50 (,)</text>\n",
       "</g>\n",
       "<!-- 48&#45;&gt;50 -->\n",
       "<g id=\"edge51\" class=\"edge\">\n",
       "<title>48&#45;&gt;50</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1104.2325,-608.6446C1097.5188,-603.2809 1090.4786,-597.1878 1084.5,-591 1077.3252,-583.5742 1070.3215,-574.7681 1064.3213,-566.5811\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1067.1175,-564.4739 1058.4666,-558.3573 1061.415,-568.5336 1067.1175,-564.4739\"/>\n",
       "<text text-anchor=\"middle\" x=\"1105\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 51 -->\n",
       "<g id=\"node54\" class=\"node\">\n",
       "<title>51</title>\n",
       "<text text-anchor=\"middle\" x=\"1129.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">51 (Tan)</text>\n",
       "</g>\n",
       "<!-- 48&#45;&gt;51 -->\n",
       "<g id=\"edge52\" class=\"edge\">\n",
       "<title>48&#45;&gt;51</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1129.5,-608.9735C1129.5,-597.1918 1129.5,-581.5607 1129.5,-568.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1133.0001,-568.0033 1129.5,-558.0034 1126.0001,-568.0034 1133.0001,-568.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1151\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">appos</text>\n",
       "</g>\n",
       "<!-- 57 -->\n",
       "<g id=\"node55\" class=\"node\">\n",
       "<title>57</title>\n",
       "<text text-anchor=\"middle\" x=\"1237.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">57 (lengkap)</text>\n",
       "</g>\n",
       "<!-- 48&#45;&gt;57 -->\n",
       "<g id=\"edge53\" class=\"edge\">\n",
       "<title>48&#45;&gt;57</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1153.5634,-608.8816C1160.9712,-603.2203 1169.1176,-596.909 1176.5,-591 1187.0446,-582.56 1198.4396,-573.1216 1208.4947,-564.6808\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1210.8375,-567.2836 1216.2295,-558.1635 1206.327,-561.9305 1210.8375,-567.2836\"/>\n",
       "<text text-anchor=\"middle\" x=\"1213.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">amod</text>\n",
       "</g>\n",
       "<!-- 52 -->\n",
       "<g id=\"node56\" class=\"node\">\n",
       "<title>52</title>\n",
       "<text text-anchor=\"middle\" x=\"1061.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">52 (Sri)</text>\n",
       "</g>\n",
       "<!-- 51&#45;&gt;52 -->\n",
       "<g id=\"edge54\" class=\"edge\">\n",
       "<title>51&#45;&gt;52</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1115.4104,-521.9735C1105.7457,-509.6085 1092.7672,-493.0036 1081.9662,-479.1847\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1084.4874,-476.7269 1075.5716,-471.0034 1078.9722,-481.0376 1084.4874,-476.7269\"/>\n",
       "<text text-anchor=\"middle\" x=\"1112.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 55 -->\n",
       "<g id=\"node59\" class=\"node\">\n",
       "<title>55</title>\n",
       "<text text-anchor=\"middle\" x=\"1157.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">55 (yang)</text>\n",
       "</g>\n",
       "<!-- 57&#45;&gt;55 -->\n",
       "<g id=\"edge57\" class=\"edge\">\n",
       "<title>57&#45;&gt;55</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1220.9239,-521.9735C1209.3393,-509.3752 1193.7073,-492.3755 1180.8614,-478.4055\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1183.3999,-475.9953 1174.0548,-471.0034 1178.2472,-480.7335 1183.3999,-475.9953\"/>\n",
       "<text text-anchor=\"middle\" x=\"1222.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 56 -->\n",
       "<g id=\"node60\" class=\"node\">\n",
       "<title>56</title>\n",
       "<text text-anchor=\"middle\" x=\"1260.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">56 (tidak)</text>\n",
       "</g>\n",
       "<!-- 57&#45;&gt;56 -->\n",
       "<g id=\"edge58\" class=\"edge\">\n",
       "<title>57&#45;&gt;56</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1242.2656,-521.9735C1245.4112,-510.0751 1249.5946,-494.2508 1253.161,-480.7606\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1256.5683,-481.5658 1255.7405,-471.0034 1249.8008,-479.7766 1256.5683,-481.5658\"/>\n",
       "<text text-anchor=\"middle\" x=\"1279\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">advmod</text>\n",
       "</g>\n",
       "<!-- 59 -->\n",
       "<g id=\"node61\" class=\"node\">\n",
       "<title>59</title>\n",
       "<text text-anchor=\"middle\" x=\"1395.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">59 (mengelirukan)</text>\n",
       "</g>\n",
       "<!-- 57&#45;&gt;59 -->\n",
       "<g id=\"edge59\" class=\"edge\">\n",
       "<title>57&#45;&gt;59</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1276.2525,-521.9847C1287.7136,-516.4326 1300.2111,-510.1557 1311.5,-504 1327.1293,-495.4776 1343.9729,-485.4298 1358.4444,-476.5159\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1360.4638,-479.382 1367.1163,-471.1361 1356.7736,-473.4337 1360.4638,-479.382\"/>\n",
       "<text text-anchor=\"middle\" x=\"1360.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">ccomp</text>\n",
       "</g>\n",
       "<!-- 53 -->\n",
       "<g id=\"node57\" class=\"node\">\n",
       "<title>53</title>\n",
       "<text text-anchor=\"middle\" x=\"1061.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">53 (Muhyiddin)</text>\n",
       "</g>\n",
       "<!-- 52&#45;&gt;53 -->\n",
       "<g id=\"edge55\" class=\"edge\">\n",
       "<title>52&#45;&gt;53</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1061.5,-434.9735C1061.5,-423.1918 1061.5,-407.5607 1061.5,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1065.0001,-394.0033 1061.5,-384.0034 1058.0001,-394.0034 1065.0001,-394.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1073.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 54 -->\n",
       "<g id=\"node58\" class=\"node\">\n",
       "<title>54</title>\n",
       "<text text-anchor=\"middle\" x=\"1061.5\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">54 (Yassin)</text>\n",
       "</g>\n",
       "<!-- 53&#45;&gt;54 -->\n",
       "<g id=\"edge56\" class=\"edge\">\n",
       "<title>53&#45;&gt;54</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1061.5,-347.9735C1061.5,-336.1918 1061.5,-320.5607 1061.5,-307.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1065.0001,-307.0033 1061.5,-297.0034 1058.0001,-307.0034 1065.0001,-307.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1073.5\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 58 -->\n",
       "<g id=\"node62\" class=\"node\">\n",
       "<title>58</title>\n",
       "<text text-anchor=\"middle\" x=\"1339.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">58 (untuk)</text>\n",
       "</g>\n",
       "<!-- 59&#45;&gt;58 -->\n",
       "<g id=\"edge60\" class=\"edge\">\n",
       "<title>59&#45;&gt;58</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1383.8968,-434.9735C1376.0128,-422.7252 1365.4509,-406.3165 1356.6067,-392.5766\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1359.4439,-390.5176 1351.0884,-384.0034 1353.5578,-394.3064 1359.4439,-390.5176\"/>\n",
       "<text text-anchor=\"middle\" x=\"1386.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">case</text>\n",
       "</g>\n",
       "<!-- 60 -->\n",
       "<g id=\"node63\" class=\"node\">\n",
       "<title>60</title>\n",
       "<text text-anchor=\"middle\" x=\"1450.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">60 (rakyat)</text>\n",
       "</g>\n",
       "<!-- 59&#45;&gt;60 -->\n",
       "<g id=\"edge61\" class=\"edge\">\n",
       "<title>59&#45;&gt;60</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1406.896,-434.9735C1414.6393,-422.7252 1425.0126,-406.3165 1433.6987,-392.5766\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1436.7333,-394.3262 1439.1186,-384.0034 1430.8165,-390.5857 1436.7333,-394.3262\"/>\n",
       "<text text-anchor=\"middle\" x=\"1436.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 63 -->\n",
       "<g id=\"node65\" class=\"node\">\n",
       "<title>63</title>\n",
       "<text text-anchor=\"middle\" x=\"1459.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">63 (Saiful)</text>\n",
       "</g>\n",
       "<!-- 62&#45;&gt;63 -->\n",
       "<g id=\"edge62\" class=\"edge\">\n",
       "<title>62&#45;&gt;63</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1459.5,-608.9735C1459.5,-597.1918 1459.5,-581.5607 1459.5,-568.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1463.0001,-568.0033 1459.5,-558.0034 1456.0001,-568.0034 1463.0001,-568.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1471.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 66 -->\n",
       "<g id=\"node69\" class=\"node\">\n",
       "<title>66</title>\n",
       "<text text-anchor=\"middle\" x=\"1614.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">66 (beliau)</text>\n",
       "</g>\n",
       "<!-- 68&#45;&gt;66 -->\n",
       "<g id=\"edge68\" class=\"edge\">\n",
       "<title>68&#45;&gt;66</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1702.5007,-608.9735C1685.9805,-596.0253 1663.5284,-578.4276 1645.4378,-564.2485\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1647.4995,-561.4175 1637.4698,-558.0034 1643.1813,-566.9269 1647.4995,-561.4175\"/>\n",
       "<text text-anchor=\"middle\" x=\"1696.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 67 -->\n",
       "<g id=\"node70\" class=\"node\">\n",
       "<title>67</title>\n",
       "<text text-anchor=\"middle\" x=\"1725.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">67 (sudah)</text>\n",
       "</g>\n",
       "<!-- 68&#45;&gt;67 -->\n",
       "<g id=\"edge69\" class=\"edge\">\n",
       "<title>68&#45;&gt;67</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1725.5,-608.9735C1725.5,-597.1918 1725.5,-581.5607 1725.5,-568.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1729.0001,-568.0033 1725.5,-558.0034 1722.0001,-568.0034 1729.0001,-568.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1754\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">advmod</text>\n",
       "</g>\n",
       "<!-- 69 -->\n",
       "<g id=\"node71\" class=\"node\">\n",
       "<title>69</title>\n",
       "<text text-anchor=\"middle\" x=\"1827.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">69 (ada)</text>\n",
       "</g>\n",
       "<!-- 68&#45;&gt;69 -->\n",
       "<g id=\"edge70\" class=\"edge\">\n",
       "<title>68&#45;&gt;69</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1761.1666,-608.8601C1769.8768,-603.6639 1778.8755,-597.5989 1786.5,-591 1794.7465,-583.8628 1802.5738,-574.9066 1809.1365,-566.4994\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1812.0405,-568.4609 1815.2465,-558.3628 1806.443,-564.2576 1812.0405,-568.4609\"/>\n",
       "<text text-anchor=\"middle\" x=\"1823\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">xcomp</text>\n",
       "</g>\n",
       "<!-- 70 -->\n",
       "<g id=\"node72\" class=\"node\">\n",
       "<title>70</title>\n",
       "<text text-anchor=\"middle\" x=\"1703.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">70 (kenyataan)</text>\n",
       "</g>\n",
       "<!-- 69&#45;&gt;70 -->\n",
       "<g id=\"edge71\" class=\"edge\">\n",
       "<title>69&#45;&gt;70</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1801.8071,-521.9735C1783.1858,-508.9086 1757.818,-491.1102 1737.5161,-476.8662\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1739.3563,-473.8817 1729.16,-471.0034 1735.3359,-479.612 1739.3563,-473.8817\"/>\n",
       "<text text-anchor=\"middle\" x=\"1783.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 73 -->\n",
       "<g id=\"node73\" class=\"node\">\n",
       "<title>73</title>\n",
       "<text text-anchor=\"middle\" x=\"1827.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">73 (Najib)</text>\n",
       "</g>\n",
       "<!-- 69&#45;&gt;73 -->\n",
       "<g id=\"edge72\" class=\"edge\">\n",
       "<title>69&#45;&gt;73</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1827.5,-521.9735C1827.5,-510.1918 1827.5,-494.5607 1827.5,-481.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1831.0001,-481.0033 1827.5,-471.0034 1824.0001,-481.0034 1831.0001,-481.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1838.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obl</text>\n",
       "</g>\n",
       "<!-- 75 -->\n",
       "<g id=\"node74\" class=\"node\">\n",
       "<title>75</title>\n",
       "<text text-anchor=\"middle\" x=\"1958.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">75 (tulisan)</text>\n",
       "</g>\n",
       "<!-- 69&#45;&gt;75 -->\n",
       "<g id=\"edge73\" class=\"edge\">\n",
       "<title>69&#45;&gt;75</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1854.6433,-521.9735C1874.4036,-508.8503 1901.3551,-490.9512 1922.8505,-476.6756\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1924.9975,-479.4513 1931.3915,-471.0034 1921.1249,-473.6201 1924.9975,-479.4513\"/>\n",
       "<text text-anchor=\"middle\" x=\"1911.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obl</text>\n",
       "</g>\n",
       "<!-- 71 -->\n",
       "<g id=\"node75\" class=\"node\">\n",
       "<title>71</title>\n",
       "<text text-anchor=\"middle\" x=\"1703.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">71 (balas)</text>\n",
       "</g>\n",
       "<!-- 70&#45;&gt;71 -->\n",
       "<g id=\"edge74\" class=\"edge\">\n",
       "<title>70&#45;&gt;71</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1703.5,-434.9735C1703.5,-423.1918 1703.5,-407.5607 1703.5,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1707.0001,-394.0033 1703.5,-384.0034 1700.0001,-394.0034 1707.0001,-394.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1726.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">ccomp</text>\n",
       "</g>\n",
       "<!-- 72 -->\n",
       "<g id=\"node76\" class=\"node\">\n",
       "<title>72</title>\n",
       "<text text-anchor=\"middle\" x=\"1824.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">72 (daripada)</text>\n",
       "</g>\n",
       "<!-- 73&#45;&gt;72 -->\n",
       "<g id=\"edge75\" class=\"edge\">\n",
       "<title>73&#45;&gt;72</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1826.8784,-434.9735C1826.4721,-423.1918 1825.9331,-407.5607 1825.471,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1828.9634,-393.8768 1825.1208,-384.0034 1821.9676,-394.1181 1828.9634,-393.8768\"/>\n",
       "<text text-anchor=\"middle\" x=\"1841.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">case</text>\n",
       "</g>\n",
       "<!-- 74 -->\n",
       "<g id=\"node77\" class=\"node\">\n",
       "<title>74</title>\n",
       "<text text-anchor=\"middle\" x=\"1958.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">74 (mengenai)</text>\n",
       "</g>\n",
       "<!-- 75&#45;&gt;74 -->\n",
       "<g id=\"edge76\" class=\"edge\">\n",
       "<title>75&#45;&gt;74</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1958.5,-434.9735C1958.5,-423.1918 1958.5,-407.5607 1958.5,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1962.0001,-394.0033 1958.5,-384.0034 1955.0001,-394.0034 1962.0001,-394.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1974.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">case</text>\n",
       "</g>\n",
       "<!-- 77 -->\n",
       "<g id=\"node78\" class=\"node\">\n",
       "<title>77</title>\n",
       "<text text-anchor=\"middle\" x=\"2100.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">77 (berhubung)</text>\n",
       "</g>\n",
       "<!-- 75&#45;&gt;77 -->\n",
       "<g id=\"edge77\" class=\"edge\">\n",
       "<title>75&#45;&gt;77</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1987.9225,-434.9735C2009.5325,-421.7336 2039.0767,-403.6326 2062.477,-389.2958\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2064.4168,-392.212 2071.1152,-384.0034 2060.7598,-386.2432 2064.4168,-392.212\"/>\n",
       "<text text-anchor=\"middle\" x=\"2048\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">acl</text>\n",
       "</g>\n",
       "<!-- 76 -->\n",
       "<g id=\"node79\" class=\"node\">\n",
       "<title>76</title>\n",
       "<text text-anchor=\"middle\" x=\"2056.5\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">76 (beliau)</text>\n",
       "</g>\n",
       "<!-- 77&#45;&gt;76 -->\n",
       "<g id=\"edge78\" class=\"edge\">\n",
       "<title>77&#45;&gt;76</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2090.7281,-347.7409C2087.7317,-342.0738 2084.4486,-335.7946 2081.5,-330 2077.5577,-322.2525 2073.366,-313.793 2069.564,-306.0307\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2072.6911,-304.4579 2065.1629,-297.0029 2066.399,-307.5254 2072.6911,-304.4579\"/>\n",
       "<text text-anchor=\"middle\" x=\"2101.5\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 78 -->\n",
       "<g id=\"node80\" class=\"node\">\n",
       "<title>78</title>\n",
       "<text text-anchor=\"middle\" x=\"2166.5\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">78 (kesan)</text>\n",
       "</g>\n",
       "<!-- 77&#45;&gt;78 -->\n",
       "<g id=\"edge79\" class=\"edge\">\n",
       "<title>77&#45;&gt;78</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2114.1752,-347.9735C2123.5556,-335.6085 2136.1525,-319.0036 2146.6357,-305.1847\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2149.5868,-307.0857 2152.8423,-297.0034 2144.0099,-302.8549 2149.5868,-307.0857\"/>\n",
       "<text text-anchor=\"middle\" x=\"2147.5\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 79 -->\n",
       "<g id=\"node81\" class=\"node\">\n",
       "<title>79</title>\n",
       "<text text-anchor=\"middle\" x=\"2121.5\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">79 (positif)</text>\n",
       "</g>\n",
       "<!-- 78&#45;&gt;79 -->\n",
       "<g id=\"edge80\" class=\"edge\">\n",
       "<title>78&#45;&gt;79</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2157.176,-260.9735C2150.9009,-248.8418 2142.5148,-232.6287 2135.4499,-218.9698\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2138.5151,-217.2776 2130.8121,-210.0034 2132.2976,-220.4935 2138.5151,-217.2776\"/>\n",
       "<text text-anchor=\"middle\" x=\"2166.5\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">amod</text>\n",
       "</g>\n",
       "<!-- 80 -->\n",
       "<g id=\"node82\" class=\"node\">\n",
       "<title>80</title>\n",
       "<text text-anchor=\"middle\" x=\"2239.5\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">80 (sekatan)</text>\n",
       "</g>\n",
       "<!-- 78&#45;&gt;80 -->\n",
       "<g id=\"edge81\" class=\"edge\">\n",
       "<title>78&#45;&gt;80</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2181.6256,-260.9735C2192.0988,-248.4919 2206.197,-231.6899 2217.8565,-217.7944\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2220.6471,-219.9136 2224.3937,-210.0034 2215.2847,-215.4142 2220.6471,-219.9136\"/>\n",
       "<text text-anchor=\"middle\" x=\"2243.5\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">compound</text>\n",
       "</g>\n",
       "<!-- 81 -->\n",
       "<g id=\"node83\" class=\"node\">\n",
       "<title>81</title>\n",
       "<text text-anchor=\"middle\" x=\"2239.5\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">81 (pergerakan)</text>\n",
       "</g>\n",
       "<!-- 80&#45;&gt;81 -->\n",
       "<g id=\"edge82\" class=\"edge\">\n",
       "<title>80&#45;&gt;81</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2239.5,-173.9735C2239.5,-162.1918 2239.5,-146.5607 2239.5,-133.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2243.0001,-133.0033 2239.5,-123.0034 2236.0001,-133.0034 2243.0001,-133.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"2276.5\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">compound</text>\n",
       "</g>\n",
       "<!-- 82 -->\n",
       "<g id=\"node84\" class=\"node\">\n",
       "<title>82</title>\n",
       "<text text-anchor=\"middle\" x=\"2239.5\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">82 (penuh)</text>\n",
       "</g>\n",
       "<!-- 81&#45;&gt;82 -->\n",
       "<g id=\"edge83\" class=\"edge\">\n",
       "<title>81&#45;&gt;82</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2239.5,-86.9735C2239.5,-75.1918 2239.5,-59.5607 2239.5,-46.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2243.0001,-46.0033 2239.5,-36.0034 2236.0001,-46.0034 2243.0001,-46.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"2259.5\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">amod</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.files.Source at 0x7f56a89d7940>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string = 'KUALA LUMPUR: Ketua Penerangan BERSATU, Datuk Wan Saiful Wan Jan membidas kenyataan Datuk Seri Najib Razak dan Ketua Pemuda UMNO, Datuk Dr Asyraf Wajdi Dusuki yang mempertikaikan tindakan kerajaan melaksanakan sekatan pergerakan penuh. Beliau berkata, Najib dan Asyraf Wajdi sengaja memetik kenyataan Perdana Menteri, Tan Sri Muhyiddin Yassin yang tidak lengkap untuk mengelirukan rakyat. Wan Saiful berkata, beliau sudah menjangka ada kenyataan balas daripada Najib mengenai tulisan beliau berhubung kesan positif sekatan pergerakan penuh.'\n",
    "sequence = transformer_textcleaning(string)[1]\n",
    "parsed_sequence, segment_sequence, mask_sequence, xlnet_sequence = parse_X(sequence)\n",
    "h, t = sess.run([model.heads_seq, model.tags_seq],\n",
    "        feed_dict = {\n",
    "            model.words: [parsed_sequence],\n",
    "            model.segment_ids: [segment_sequence],\n",
    "            model.input_masks: [mask_sequence],\n",
    "        },\n",
    ")\n",
    "h = h[0] - 2\n",
    "t = [idx2tag[d] for d in t[0]]\n",
    "merged_h = merge_sentencepiece_tokens_tagging(xlnet_sequence, h)\n",
    "merged_t = merge_sentencepiece_tokens_tagging(xlnet_sequence, t)\n",
    "tagging = list(zip(merged_t[0], merged_t[1]))\n",
    "indexing = list(zip(merged_h[0], merged_h[1]))\n",
    "dep = dependency_graph(tagging, indexing)\n",
    "dep.to_graphvis()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'alxlnet-base-dependency/model.ckpt'"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'alxlnet-base-dependency/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "kwargs = dict(\n",
    "      is_training=False,\n",
    "      use_tpu=False,\n",
    "      use_bfloat16=False,\n",
    "      dropout=0.0,\n",
    "      dropatt=0.0,\n",
    "      init='normal',\n",
    "      init_range=0.1,\n",
    "      init_std=0.05,\n",
    "      clamp_len=-1)\n",
    "\n",
    "xlnet_parameters = xlnet.RunConfig(**kwargs)\n",
    "xlnet_config = xlnet.XLNetConfig(\n",
    "    json_path = 'alxlnet-base-2020-04-10/config.json'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:memory input None\n",
      "INFO:tensorflow:Use float type <dtype: 'float32'>\n"
     ]
    }
   ],
   "source": [
    "learning_rate = 2e-5\n",
    "hidden_size_word = 256\n",
    "\n",
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model(learning_rate, hidden_size_word)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from alxlnet-base-dependency/model.ckpt\n"
     ]
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.restore(sess, 'alxlnet-base-dependency/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pred2label(pred):\n",
    "    out = []\n",
    "    for pred_i in pred:\n",
    "        out_i = []\n",
    "        for p in pred_i:\n",
    "            out_i.append(idx2tag[p])\n",
    "        out.append(out_i)\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(heads_pred, types_pred, heads, types, lengths,\n",
    "             symbolic_root=False, symbolic_end=False):\n",
    "    batch_size, _ = heads_pred.shape\n",
    "    ucorr = 0.\n",
    "    lcorr = 0.\n",
    "    total = 0.\n",
    "    ucomplete_match = 0.\n",
    "    lcomplete_match = 0.\n",
    "\n",
    "    corr_root = 0.\n",
    "    total_root = 0.\n",
    "    start = 1 if symbolic_root else 0\n",
    "    end = 1 if symbolic_end else 0\n",
    "    for i in range(batch_size):\n",
    "        ucm = 1.\n",
    "        lcm = 1.\n",
    "        for j in range(start, lengths[i] - end):\n",
    "\n",
    "            total += 1\n",
    "            if heads[i, j] == heads_pred[i, j]:\n",
    "                ucorr += 1\n",
    "                if types[i, j] == types_pred[i, j]:\n",
    "                    lcorr += 1\n",
    "                else:\n",
    "                    lcm = 0\n",
    "            else:\n",
    "                ucm = 0\n",
    "                lcm = 0\n",
    "\n",
    "            if heads[i, j] == 0:\n",
    "                total_root += 1\n",
    "                corr_root += 1 if heads_pred[i, j] == 0 else 0\n",
    "\n",
    "        ucomplete_match += ucm\n",
    "        lcomplete_match += lcm\n",
    "    \n",
    "    return ucorr / total, lcorr / total, corr_root / total_root"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1250/1250 [02:00<00:00, 10.39it/s]\n"
     ]
    }
   ],
   "source": [
    "arcs, types, roots = [], [], []\n",
    "real_Y, predict_Y = [], []\n",
    "\n",
    "for i in tqdm(range(0, len(test_X), batch_size)):\n",
    "    index = min(i + batch_size, len(test_X))\n",
    "    batch_x = test_X[i: index]\n",
    "    batch_x = pad_sequences(batch_x,padding='post')\n",
    "    batch_y = test_Y[i: index]\n",
    "    batch_y = pad_sequences(batch_y,padding='post')\n",
    "    batch_depends = test_depends[i: index]\n",
    "    batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "    batch_segments = test_segments[i: index]\n",
    "    batch_segments = pad_sequences(batch_segments, padding='post', value = 4)\n",
    "    batch_masks = test_masks[i: index]\n",
    "    batch_masks = pad_sequences(batch_masks, padding='post', value = 1)\n",
    "    \n",
    "    tags_seq, heads = sess.run(\n",
    "        [model.logits, model.heads_seq],\n",
    "        feed_dict = {\n",
    "            model.words: batch_x,\n",
    "            model.segment_ids: batch_segments,\n",
    "            model.input_masks: batch_masks\n",
    "        },\n",
    "    )\n",
    "    \n",
    "    arc_accuracy, type_accuracy, root_accuracy = evaluate(heads - 1, tags_seq, batch_depends - 1, batch_y, \n",
    "            np.count_nonzero(batch_x, axis = 1))\n",
    "    arcs.append(arc_accuracy)\n",
    "    types.append(type_accuracy)\n",
    "    roots.append(root_accuracy)\n",
    "    predicted = pred2label(tags_seq)\n",
    "    real = pred2label(batch_y)\n",
    "    predict_Y.extend(predicted)\n",
    "    real_Y.extend(real)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp_real_Y = []\n",
    "for r in real_Y:\n",
    "    temp_real_Y.extend(r)\n",
    "    \n",
    "temp_predict_Y = []\n",
    "for r in predict_Y:\n",
    "    temp_predict_Y.extend(r)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "               precision    recall  f1-score   support\n",
      "\n",
      "          PAD    0.99976   1.00000   0.99988    339805\n",
      "            X    1.00000   0.99936   0.99968     62631\n",
      "          acl    0.83797   0.82854   0.83323      3202\n",
      "        advcl    0.63223   0.66865   0.64993      1684\n",
      "       advmod    0.95967   0.93403   0.94668      6700\n",
      "         amod    0.90195   0.90054   0.90124      4464\n",
      "        appos    0.85624   0.74838   0.79869      3088\n",
      "         case    0.98172   0.98093   0.98133     11117\n",
      "           cc    0.98209   0.97993   0.98101      3637\n",
      "        ccomp    0.50143   0.49157   0.49645       356\n",
      "     compound    0.91041   0.92329   0.91681     11381\n",
      "compound:plur    0.55000   0.66667   0.60274        33\n",
      "         conj    0.89766   0.88735   0.89248      5140\n",
      "          cop    0.96975   0.97302   0.97138       593\n",
      "        csubj    0.25000   0.16667   0.20000         6\n",
      "   csubj:pass    0.00000   0.00000   0.00000         1\n",
      "          dep    0.65012   0.72576   0.68586       361\n",
      "          det    0.94992   0.91641   0.93286      3912\n",
      "        fixed    0.87597   0.77397   0.82182       146\n",
      "         flat    0.95129   0.97231   0.96169     18638\n",
      "         iobj    1.00000   0.25000   0.40000         4\n",
      "         mark    0.91882   0.92439   0.92160      1812\n",
      "         nmod    0.85028   0.84376   0.84701      4429\n",
      "        nsubj    0.83595   0.87600   0.85551      6992\n",
      "   nsubj:pass    0.82935   0.78216   0.80506      1951\n",
      "       nummod    0.97242   0.95072   0.96145      4302\n",
      "          obj    0.90211   0.91702   0.90950      6351\n",
      "          obl    0.87606   0.85241   0.86408      5075\n",
      "    parataxis    0.46886   0.31762   0.37870       403\n",
      "        punct    0.99703   0.99631   0.99667     20881\n",
      "         root    0.89346   0.91070   0.90200     10000\n",
      "        xcomp    0.75430   0.74016   0.74716      1601\n",
      "\n",
      "     accuracy                        0.97964    540696\n",
      "    macro avg    0.81115   0.77808   0.78633    540696\n",
      " weighted avg    0.97957   0.97964   0.97954    540696\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(temp_real_Y, temp_predict_Y, digits = 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "arc accuracy: 0.8492916621693761\n",
      "types accuracy: 0.8281206797454291\n",
      "root accuracy: 0.9209962611210157\n"
     ]
    }
   ],
   "source": [
    "print('arc accuracy:', np.mean(arcs))\n",
    "print('types accuracy:', np.mean(types))\n",
    "print('root accuracy:', np.mean(roots))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "strings = ','.join(\n",
    "    [\n",
    "        n.name\n",
    "        for n in tf.get_default_graph().as_graph_def().node\n",
    "        if ('Variable' in n.op\n",
    "        or 'Placeholder' in n.name\n",
    "        or '_seq' in n.name\n",
    "        or 'logits' in n.name\n",
    "        or 'alphas' in n.name\n",
    "        or 'self/Softmax' in n.name)\n",
    "        and 'Adam' not in n.name\n",
    "        and 'beta' not in n.name\n",
    "        and 'global_step' not in n.name\n",
    "        and 'adam' not in n.name\n",
    "        and 'gradients/bert' not in n.name\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            'directory: %s' % model_dir\n",
    "        )\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "\n",
    "    absolute_model_dir = '/'.join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + '/frozen_model.pb'\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph = tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(\n",
    "            input_checkpoint + '.meta', clear_devices = clear_devices\n",
    "        )\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names.split(','),\n",
    "        )\n",
    "        with tf.gfile.GFile(output_graph, 'wb') as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print('%d ops in the final graph.' % len(output_graph_def.node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from alxlnet-base-dependency/model.ckpt\n",
      "WARNING:tensorflow:From <ipython-input-35-9a7215a4e58a>:23: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n",
      "INFO:tensorflow:Froze 34 variables.\n",
      "INFO:tensorflow:Converted 34 variables to const ops.\n",
      "7923 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph('alxlnet-base-dependency', strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "transforms = ['add_default_attributes',\n",
    "             'remove_nodes(op=Identity, op=CheckNumerics, op=Dropout)',\n",
    "             'fold_batch_norms',\n",
    "             'fold_old_batch_norms',\n",
    "             'quantize_weights(fallback_min=-10, fallback_max=10)',\n",
    "             'strip_unused_nodes',\n",
    "             'sort_by_execution_order']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-38-2caa363a97f9>:6: FastGFile.__init__ (from tensorflow.python.platform.gfile) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.gfile.GFile.\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.tools.graph_transforms import TransformGraph\n",
    "tf.set_random_seed(0)\n",
    "\n",
    "pb = 'alxlnet-base-dependency/frozen_model.pb'\n",
    "input_graph_def = tf.GraphDef()\n",
    "with tf.gfile.FastGFile(pb, 'rb') as f:\n",
    "    input_graph_def.ParseFromString(f.read())\n",
    "\n",
    "if 'bert' in pb:\n",
    "    inputs = ['Placeholder']\n",
    "    a = ['dense/BiasAdd']\n",
    "if 'xlnet' in pb:\n",
    "    inputs = ['Placeholder', 'Placeholder_1', 'Placeholder_2']\n",
    "    a = ['transpose_3']\n",
    "\n",
    "transformed_graph_def = TransformGraph(input_graph_def, \n",
    "                                       inputs,\n",
    "                                       ['logits', 'heads_seq'] + a, transforms)\n",
    "\n",
    "with tf.gfile.GFile(f'{pb}.quantized', 'wb') as f:\n",
    "    f.write(transformed_graph_def.SerializeToString())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
